import tensorflow as tf
import pandas as pd
import numpy as np
import keras
import sklearn
import itertools
import matplotlib.pyplot as plt

X_train = pd.read_csv("X_train.csv")
X_test = pd.read_csv("X_test.csv")
y_train = pd.read_csv("y_train.csv")
y_test = pd.read_csv("y_test.csv")
y_multi_train = pd.read_csv("y_multi_train.csv")
y_multi_test = pd.read_csv("y_multi_test.csv")

from sklearn.preprocessing import StandardScaler
ss = StandardScaler()
X_train = ss.fit_transform(X_train)
X_test = ss.fit_transform(X_test)

#dictionaries for final plots
res_log=dict()
res_rf_b=dict()
res_xgb_b=dict()
res_nn_b=dict()
res_tree=dict()
res_rf_m=dict()
res_xgb_m=dict()
res_nn_m=dict()

Noise range 0.5

def modify_dataset(dataset, variables_to_poison, noise_factor):
    poisoned_dataset = dataset.copy()
    num_samples, num_variables = poisoned_dataset.shape
    for var_index in variables_to_poison:
        noise = np.random.normal(scale=noise_factor, size=num_samples)
        poisoned_dataset[:, var_index] += noise
    return poisoned_dataset

variables_to_poison = [2, 6, 8, 9, 11, 31, 33, 34, 39, 40]
noise1 = 1
X_train_mod1 = modify_dataset(X_train, variables_to_poison, noise1)

X_train_mod1

array([[ 1.79214449, -0.2082886 , -0.05701697, ..., -0.47506312,
        -0.73447607, -0.12070382],
       [-0.57638416,  1.78686151,  1.3891409 , ..., -1.07749554,
        -1.73937336, -0.12070382],
       [-1.0569863 , -0.20828827,  0.02590353, ...,  3.2904489 ,
         3.30201687, -0.12070382],
       ...,
       [-0.47187522, -0.06694436,  2.39135366, ..., -0.15729827,
        -1.36659372, -0.12070382],
       [-0.1666061 , -0.18783648, -1.04653667, ...,  1.07142868,
         2.04014546, -0.12070382],
       [-0.6757627 , -0.20011705, -0.68488861, ..., -0.72946225,
        -0.92926595, -0.12070382]])

from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod1, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod1, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['1']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.8946699910617679
score on train: 0.8940103110699184
              precision    recall  f1-score   support

           0       0.94      0.76      0.84     30772
           1       0.88      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.91      0.87      0.88     85028
weighted avg       0.90      0.89      0.89     85028

from sklearn.metrics import roc_auc_score,accuracy_score
from sklearn.metrics import roc_curve
# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9581

from sklearn.ensemble import RandomForestClassifier
import pickle
rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod1, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod1, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['1']=acc

<ipython-input-8-68bbbc7f51d8>:4: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod1, y_train)

score on test: 0.9373030060685892
score on train: 0.9538956148989167
              precision    recall  f1-score   support

           0       0.90      0.94      0.92     30772
           1       0.96      0.94      0.95     54256

    accuracy                           0.94     85028
   macro avg       0.93      0.94      0.93     85028
weighted avg       0.94      0.94      0.94     85028

# ROC curve and AUC
rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9724

import xgboost as xgb
from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
import matplotlib.pyplot as plt
xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod1, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod1, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['1']=acc

score on test: 0.971915133838265
score on train: 0.9921508428430748
              precision    recall  f1-score   support

           0       0.96      0.96      0.96     30772
           1       0.98      0.98      0.98     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.97      0.97     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9966

from keras import backend as K
def recall_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall
def precision_m(y_true, y_pred):
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision
def f1_m(y_true, y_pred):
    precision = precision_m(y_true, y_pred)
    recall = recall_m(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

import keras
from keras import layers
from keras import models
from keras import optimizers
from keras import losses
from keras import callbacks
from keras import metrics

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod1, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod1,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores = model.evaluate(X_test, y_test, verbose=0)
res_nn_b['1']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 4s 2ms/step - loss: 0.2019 - accuracy: 0.9074 - f1_m: 0.9293 - precision_m: 0.9087 - recall_m: 0.9526
Epoch 2/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.1579 - accuracy: 0.9273 - f1_m: 0.9433 - precision_m: 0.9363 - recall_m: 0.9514
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1401 - accuracy: 0.9359 - f1_m: 0.9497 - precision_m: 0.9486 - recall_m: 0.9516
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1240 - accuracy: 0.9453 - f1_m: 0.9570 - precision_m: 0.9569 - recall_m: 0.9578
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1021 - accuracy: 0.9572 - f1_m: 0.9664 - precision_m: 0.9663 - recall_m: 0.9670
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0918 - accuracy: 0.9616 - f1_m: 0.9698 - precision_m: 0.9701 - recall_m: 0.9700
Epoch 7/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0854 - accuracy: 0.9643 - f1_m: 0.9719 - precision_m: 0.9731 - recall_m: 0.9711
Epoch 8/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0816 - accuracy: 0.9663 - f1_m: 0.9735 - precision_m: 0.9753 - recall_m: 0.9721
Epoch 9/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0792 - accuracy: 0.9668 - f1_m: 0.9738 - precision_m: 0.9759 - recall_m: 0.9721
Epoch 10/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0776 - accuracy: 0.9679 - f1_m: 0.9747 - precision_m: 0.9773 - recall_m: 0.9724
Epoch 11/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0752 - accuracy: 0.9690 - f1_m: 0.9755 - precision_m: 0.9782 - recall_m: 0.9731
Epoch 12/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0729 - accuracy: 0.9697 - f1_m: 0.9761 - precision_m: 0.9789 - recall_m: 0.9736
Epoch 13/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0721 - accuracy: 0.9697 - f1_m: 0.9761 - precision_m: 0.9787 - recall_m: 0.9738
Epoch 14/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0706 - accuracy: 0.9705 - f1_m: 0.9767 - precision_m: 0.9796 - recall_m: 0.9741
Epoch 15/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0702 - accuracy: 0.9702 - f1_m: 0.9765 - precision_m: 0.9796 - recall_m: 0.9736
Epoch 16/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0689 - accuracy: 0.9713 - f1_m: 0.9773 - precision_m: 0.9804 - recall_m: 0.9746
Epoch 17/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0674 - accuracy: 0.9719 - f1_m: 0.9778 - precision_m: 0.9811 - recall_m: 0.9747
Epoch 18/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0664 - accuracy: 0.9725 - f1_m: 0.9783 - precision_m: 0.9813 - recall_m: 0.9755
Epoch 19/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0654 - accuracy: 0.9727 - f1_m: 0.9784 - precision_m: 0.9818 - recall_m: 0.9754
Epoch 20/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0650 - accuracy: 0.9732 - f1_m: 0.9788 - precision_m: 0.9822 - recall_m: 0.9758
Epoch 21/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0638 - accuracy: 0.9733 - f1_m: 0.9790 - precision_m: 0.9823 - recall_m: 0.9759
Epoch 22/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0638 - accuracy: 0.9736 - f1_m: 0.9792 - precision_m: 0.9822 - recall_m: 0.9764
Epoch 23/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0618 - accuracy: 0.9740 - f1_m: 0.9795 - precision_m: 0.9826 - recall_m: 0.9766
Epoch 24/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0613 - accuracy: 0.9741 - f1_m: 0.9796 - precision_m: 0.9827 - recall_m: 0.9768
Epoch 25/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0603 - accuracy: 0.9747 - f1_m: 0.9801 - precision_m: 0.9834 - recall_m: 0.9771
Epoch 26/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0605 - accuracy: 0.9747 - f1_m: 0.9800 - precision_m: 0.9833 - recall_m: 0.9770
Epoch 27/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0596 - accuracy: 0.9749 - f1_m: 0.9802 - precision_m: 0.9836 - recall_m: 0.9770
Epoch 28/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0585 - accuracy: 0.9756 - f1_m: 0.9808 - precision_m: 0.9839 - recall_m: 0.9779
Epoch 29/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0582 - accuracy: 0.9755 - f1_m: 0.9807 - precision_m: 0.9839 - recall_m: 0.9778
Epoch 30/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0573 - accuracy: 0.9761 - f1_m: 0.9811 - precision_m: 0.9844 - recall_m: 0.9782
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0573 - accuracy: 0.9760 - f1_m: 0.9811 - precision_m: 0.9843 - recall_m: 0.9781
Epoch 32/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0567 - accuracy: 0.9764 - f1_m: 0.9813 - precision_m: 0.9847 - recall_m: 0.9783
Epoch 33/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0564 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9846 - recall_m: 0.9785
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0556 - accuracy: 0.9767 - f1_m: 0.9816 - precision_m: 0.9846 - recall_m: 0.9788
Epoch 35/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0547 - accuracy: 0.9775 - f1_m: 0.9823 - precision_m: 0.9854 - recall_m: 0.9794
Epoch 36/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0541 - accuracy: 0.9771 - f1_m: 0.9820 - precision_m: 0.9851 - recall_m: 0.9791
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0533 - accuracy: 0.9779 - f1_m: 0.9826 - precision_m: 0.9856 - recall_m: 0.9797
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0535 - accuracy: 0.9776 - f1_m: 0.9823 - precision_m: 0.9855 - recall_m: 0.9794
Epoch 39/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0528 - accuracy: 0.9780 - f1_m: 0.9826 - precision_m: 0.9855 - recall_m: 0.9800
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0523 - accuracy: 0.9780 - f1_m: 0.9826 - precision_m: 0.9857 - recall_m: 0.9797
Epoch 41/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0521 - accuracy: 0.9782 - f1_m: 0.9828 - precision_m: 0.9860 - recall_m: 0.9798
Epoch 42/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0511 - accuracy: 0.9786 - f1_m: 0.9831 - precision_m: 0.9861 - recall_m: 0.9804
Epoch 43/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0510 - accuracy: 0.9785 - f1_m: 0.9830 - precision_m: 0.9860 - recall_m: 0.9802
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0507 - accuracy: 0.9789 - f1_m: 0.9833 - precision_m: 0.9863 - recall_m: 0.9806
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0507 - accuracy: 0.9787 - f1_m: 0.9832 - precision_m: 0.9861 - recall_m: 0.9805
Epoch 46/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0490 - accuracy: 0.9794 - f1_m: 0.9837 - precision_m: 0.9871 - recall_m: 0.9806
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0491 - accuracy: 0.9793 - f1_m: 0.9837 - precision_m: 0.9869 - recall_m: 0.9808
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0484 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9871 - recall_m: 0.9811
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0482 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9870 - recall_m: 0.9812
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0483 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9872 - recall_m: 0.9810
Epoch 51/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0479 - accuracy: 0.9800 - f1_m: 0.9842 - precision_m: 0.9872 - recall_m: 0.9815
Epoch 52/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0471 - accuracy: 0.9802 - f1_m: 0.9844 - precision_m: 0.9872 - recall_m: 0.9817
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0465 - accuracy: 0.9802 - f1_m: 0.9844 - precision_m: 0.9873 - recall_m: 0.9818
Epoch 54/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0462 - accuracy: 0.9808 - f1_m: 0.9848 - precision_m: 0.9876 - recall_m: 0.9822
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0461 - accuracy: 0.9807 - f1_m: 0.9848 - precision_m: 0.9876 - recall_m: 0.9823
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9809 - f1_m: 0.9849 - precision_m: 0.9878 - recall_m: 0.9823
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0463 - accuracy: 0.9806 - f1_m: 0.9847 - precision_m: 0.9875 - recall_m: 0.9821
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0457 - accuracy: 0.9811 - f1_m: 0.9852 - precision_m: 0.9878 - recall_m: 0.9828
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0444 - accuracy: 0.9815 - f1_m: 0.9854 - precision_m: 0.9880 - recall_m: 0.9830
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0443 - accuracy: 0.9816 - f1_m: 0.9855 - precision_m: 0.9881 - recall_m: 0.9831
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0445 - accuracy: 0.9813 - f1_m: 0.9853 - precision_m: 0.9879 - recall_m: 0.9828
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0440 - accuracy: 0.9816 - f1_m: 0.9855 - precision_m: 0.9881 - recall_m: 0.9830
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0438 - accuracy: 0.9819 - f1_m: 0.9857 - precision_m: 0.9886 - recall_m: 0.9830
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0428 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9883 - recall_m: 0.9835
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0423 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9885 - recall_m: 0.9834
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0422 - accuracy: 0.9821 - f1_m: 0.9859 - precision_m: 0.9884 - recall_m: 0.9835
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0423 - accuracy: 0.9822 - f1_m: 0.9860 - precision_m: 0.9883 - recall_m: 0.9839
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0422 - accuracy: 0.9821 - f1_m: 0.9859 - precision_m: 0.9885 - recall_m: 0.9835
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0416 - accuracy: 0.9827 - f1_m: 0.9863 - precision_m: 0.9891 - recall_m: 0.9838
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0417 - accuracy: 0.9826 - f1_m: 0.9863 - precision_m: 0.9889 - recall_m: 0.9838
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0417 - accuracy: 0.9826 - f1_m: 0.9863 - precision_m: 0.9890 - recall_m: 0.9837
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0402 - accuracy: 0.9832 - f1_m: 0.9868 - precision_m: 0.9892 - recall_m: 0.9845
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0416 - accuracy: 0.9827 - f1_m: 0.9863 - precision_m: 0.9888 - recall_m: 0.9840
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0412 - accuracy: 0.9828 - f1_m: 0.9865 - precision_m: 0.9889 - recall_m: 0.9842
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0406 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9895 - recall_m: 0.9844
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0397 - accuracy: 0.9834 - f1_m: 0.9870 - precision_m: 0.9892 - recall_m: 0.9849
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0391 - accuracy: 0.9838 - f1_m: 0.9873 - precision_m: 0.9897 - recall_m: 0.9850
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0395 - accuracy: 0.9833 - f1_m: 0.9868 - precision_m: 0.9893 - recall_m: 0.9845
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0383 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9899 - recall_m: 0.9854
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0387 - accuracy: 0.9838 - f1_m: 0.9872 - precision_m: 0.9894 - recall_m: 0.9852
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0382 - accuracy: 0.9840 - f1_m: 0.9874 - precision_m: 0.9895 - recall_m: 0.9855
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0379 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9899 - recall_m: 0.9853
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0379 - accuracy: 0.9841 - f1_m: 0.9875 - precision_m: 0.9897 - recall_m: 0.9855
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0378 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9899 - recall_m: 0.9853
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0372 - accuracy: 0.9844 - f1_m: 0.9877 - precision_m: 0.9900 - recall_m: 0.9855
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0371 - accuracy: 0.9847 - f1_m: 0.9879 - precision_m: 0.9901 - recall_m: 0.9859
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0363 - accuracy: 0.9849 - f1_m: 0.9881 - precision_m: 0.9903 - recall_m: 0.9861
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0367 - accuracy: 0.9847 - f1_m: 0.9879 - precision_m: 0.9903 - recall_m: 0.9858
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0363 - accuracy: 0.9850 - f1_m: 0.9882 - precision_m: 0.9902 - recall_m: 0.9863
Epoch 90/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0359 - accuracy: 0.9852 - f1_m: 0.9883 - precision_m: 0.9904 - recall_m: 0.9864
Epoch 91/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0366 - accuracy: 0.9851 - f1_m: 0.9882 - precision_m: 0.9905 - recall_m: 0.9861
Epoch 92/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0359 - accuracy: 0.9850 - f1_m: 0.9882 - precision_m: 0.9902 - recall_m: 0.9862
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0353 - accuracy: 0.9852 - f1_m: 0.9883 - precision_m: 0.9906 - recall_m: 0.9862
Epoch 94/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0344 - accuracy: 0.9856 - f1_m: 0.9886 - precision_m: 0.9907 - recall_m: 0.9867
Epoch 95/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0357 - accuracy: 0.9851 - f1_m: 0.9882 - precision_m: 0.9906 - recall_m: 0.9861
Epoch 96/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0355 - accuracy: 0.9854 - f1_m: 0.9885 - precision_m: 0.9906 - recall_m: 0.9866
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0356 - accuracy: 0.9857 - f1_m: 0.9887 - precision_m: 0.9906 - recall_m: 0.9870
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0335 - accuracy: 0.9860 - f1_m: 0.9890 - precision_m: 0.9909 - recall_m: 0.9872
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0353 - accuracy: 0.9858 - f1_m: 0.9888 - precision_m: 0.9909 - recall_m: 0.9868
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0343 - accuracy: 0.9861 - f1_m: 0.9890 - precision_m: 0.9911 - recall_m: 0.9871
2658/2658 [==============================] - 3s 936us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0833 - accuracy: 0.9731 - f1_m: 0.9783 - precision_m: 0.9840 - recall_m: 0.9737
score on test: 0.9731147289276123
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0341 - accuracy: 0.9862 - f1_m: 0.9889 - precision_m: 0.9956 - recall_m: 0.9828
score on train: 0.9862306714057922

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

attack_cat=['Normal', 'Generic', 'Exploits', 'Fuzzers', 'DoS', 'Reconnaissance', 'Analysis', 'Backdoor', 'Shellcode', 'Worms']
import numpy as np
import itertools
def plot_confusion_matrix(cm, target_names, title='Confusion matrix', cmap=None, normalize=True):
    accuracy = np.trace(cm) / float(np.sum(cm))
    misclass = 1 - accuracy
    if cmap is None:
        cmap = plt.get_cmap('Blues')
    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.2f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))
    plt.show()

from sklearn.tree import DecisionTreeClassifier
dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod1, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod1, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['1']=acc

score on test: 0.8079338570823729
score on train: 0.8272200660371893
              precision    recall  f1-score   support

           0       0.91      0.93      0.92     30772
           1       0.97      0.97      0.97     19423
           2       0.60      0.84      0.70     14709
           3       0.68      0.59      0.63      7921
           4       0.40      0.13      0.20      5357
           5       0.70      0.72      0.71      4578
           6       0.69      0.02      0.05       909
           7       0.16      0.02      0.03       780
           8       0.00      0.00      0.00       513
           9       1.00      0.06      0.11        66

    accuracy                           0.81     85028
   macro avg       0.61      0.43      0.43     85028
weighted avg       0.79      0.81      0.79     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

import yellowbrick
from yellowbrick.classifier import ROCAUC

def plot_ROC_curve(model, xtrain, ytrain, xtest, ytest):
    # Creating visualization
    visualizer = ROCAUC(model)
    # Fitting to the training data first then scoring with the test data
    visualizer.fit(xtrain, ytrain)
    visualizer.score(xtest, ytest)
    visualizer.show()
    return visualizer

plot_ROC_curve(dtree1, X_train_mod1, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod1, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod1, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['1']=acc

<ipython-input-19-7521fd7bd576>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod1, y_multi_train)

score on test: 0.7417203744648821
score on train: 0.8156983143138504
              precision    recall  f1-score   support

           0       0.64      0.97      0.77     30772
           1       1.00      0.94      0.97     19423
           2       0.73      0.61      0.67     14709
           3       0.75      0.43      0.55      7921
           4       0.62      0.01      0.02      5357
           5       0.88      0.52      0.65      4578
           6       1.00      0.00      0.01       909
           7       0.00      0.00      0.00       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.74     85028
   macro avg       0.56      0.35      0.36     85028
weighted avg       0.75      0.74      0.70     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod1, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod1, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod1, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['1']=acc

score on test: 0.8551183139671638
score on train: 0.9077854370619244
              precision    recall  f1-score   support

           0       0.95      0.97      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.67      0.80      0.73     14709
           3       0.80      0.77      0.79      7921
           4       0.41      0.38      0.39      5357
           5       0.91      0.76      0.83      4578
           6       0.67      0.09      0.17       909
           7       0.57      0.08      0.14       780
           8       0.74      0.34      0.47       513
           9       0.71      0.26      0.38        66

    accuracy                           0.86     85028
   macro avg       0.74      0.54      0.58     85028
weighted avg       0.86      0.86      0.85     85028

plot_ROC_curve(xgb2, X_train_mod1, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

from keras import metrics
model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod1,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod1,y_multi_train)[1]))
scores = model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['1']=scores[1]
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_4 (Dense)             (None, 128)               5504      
                                                                 
 dense_5 (Dense)             (None, 64)                8256      
                                                                 
 dense_6 (Dense)             (None, 64)                4160      
                                                                 
 dense_7 (Dense)             (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.7637 - accuracy: 0.7211 - f1_m: 1.6319 - precision_m: 6471460.5000 - recall_m: 1.6575
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5989 - accuracy: 0.7755 - f1_m: 1.4359 - precision_m: 1.3248 - recall_m: 1.5754
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5541 - accuracy: 0.7908 - f1_m: 1.3753 - precision_m: 1.2517 - recall_m: 1.5333
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5307 - accuracy: 0.7980 - f1_m: 1.3421 - precision_m: 1.2131 - recall_m: 1.5090
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5172 - accuracy: 0.8027 - f1_m: 1.3278 - precision_m: 1.1965 - recall_m: 1.4987
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5037 - accuracy: 0.8077 - f1_m: 1.3137 - precision_m: 1.1789 - recall_m: 1.4901
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4933 - accuracy: 0.8100 - f1_m: 1.3016 - precision_m: 1.1654 - recall_m: 1.4802
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4846 - accuracy: 0.8141 - f1_m: 1.2919 - precision_m: 1.1537 - recall_m: 1.4743
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4760 - accuracy: 0.8171 - f1_m: 1.2838 - precision_m: 1.1434 - recall_m: 1.4700
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4679 - accuracy: 0.8205 - f1_m: 1.2748 - precision_m: 1.1340 - recall_m: 1.4619
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4581 - accuracy: 0.8239 - f1_m: 1.2682 - precision_m: 1.1251 - recall_m: 1.4593
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4510 - accuracy: 0.8272 - f1_m: 1.2589 - precision_m: 1.1152 - recall_m: 1.4509
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4449 - accuracy: 0.8305 - f1_m: 1.2534 - precision_m: 1.1097 - recall_m: 1.4462
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4381 - accuracy: 0.8321 - f1_m: 1.2504 - precision_m: 1.1065 - recall_m: 1.4438
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4334 - accuracy: 0.8338 - f1_m: 1.2471 - precision_m: 1.1021 - recall_m: 1.4423
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4278 - accuracy: 0.8356 - f1_m: 1.2408 - precision_m: 1.0940 - recall_m: 1.4387
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4237 - accuracy: 0.8376 - f1_m: 1.2389 - precision_m: 1.0923 - recall_m: 1.4374
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4195 - accuracy: 0.8381 - f1_m: 1.2356 - precision_m: 1.0879 - recall_m: 1.4358
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4166 - accuracy: 0.8395 - f1_m: 1.2340 - precision_m: 1.0857 - recall_m: 1.4351
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4140 - accuracy: 0.8400 - f1_m: 1.2325 - precision_m: 1.0841 - recall_m: 1.4339
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4096 - accuracy: 0.8416 - f1_m: 1.2298 - precision_m: 1.0808 - recall_m: 1.4325
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4071 - accuracy: 0.8428 - f1_m: 1.2270 - precision_m: 1.0779 - recall_m: 1.4299
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4043 - accuracy: 0.8438 - f1_m: 1.2221 - precision_m: 1.0728 - recall_m: 1.4255
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4016 - accuracy: 0.8446 - f1_m: 1.2211 - precision_m: 1.0711 - recall_m: 1.4258
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4007 - accuracy: 0.8449 - f1_m: 1.2229 - precision_m: 1.0735 - recall_m: 1.4266
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3992 - accuracy: 0.8451 - f1_m: 1.2193 - precision_m: 1.0699 - recall_m: 1.4231
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3962 - accuracy: 0.8462 - f1_m: 1.2166 - precision_m: 1.0662 - recall_m: 1.4224
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3950 - accuracy: 0.8474 - f1_m: 1.2142 - precision_m: 1.0637 - recall_m: 1.4200
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3935 - accuracy: 0.8472 - f1_m: 1.2155 - precision_m: 1.0655 - recall_m: 1.4209
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3925 - accuracy: 0.8472 - f1_m: 1.2165 - precision_m: 1.0656 - recall_m: 1.4230
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3900 - accuracy: 0.8484 - f1_m: 1.2143 - precision_m: 1.0634 - recall_m: 1.4212
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3890 - accuracy: 0.8487 - f1_m: 1.2123 - precision_m: 1.0615 - recall_m: 1.4185
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3885 - accuracy: 0.8484 - f1_m: 1.2129 - precision_m: 1.0623 - recall_m: 1.4196
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3871 - accuracy: 0.8490 - f1_m: 1.2095 - precision_m: 1.0581 - recall_m: 1.4173
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3853 - accuracy: 0.8499 - f1_m: 1.2078 - precision_m: 1.0565 - recall_m: 1.4150
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3850 - accuracy: 0.8496 - f1_m: 1.2096 - precision_m: 1.0586 - recall_m: 1.4162
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3842 - accuracy: 0.8503 - f1_m: 1.2055 - precision_m: 1.0538 - recall_m: 1.4142
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3825 - accuracy: 0.8509 - f1_m: 1.2064 - precision_m: 1.0548 - recall_m: 1.4145
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3808 - accuracy: 0.8513 - f1_m: 1.2056 - precision_m: 1.0542 - recall_m: 1.4134
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3808 - accuracy: 0.8514 - f1_m: 1.2048 - precision_m: 1.0539 - recall_m: 1.4120
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3802 - accuracy: 0.8517 - f1_m: 1.2041 - precision_m: 1.0523 - recall_m: 1.4128
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3780 - accuracy: 0.8519 - f1_m: 1.2040 - precision_m: 1.0524 - recall_m: 1.4121
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3775 - accuracy: 0.8530 - f1_m: 1.2020 - precision_m: 1.0498 - recall_m: 1.4113
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3759 - accuracy: 0.8527 - f1_m: 1.2003 - precision_m: 1.0476 - recall_m: 1.4103
Epoch 45/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3750 - accuracy: 0.8530 - f1_m: 1.2016 - precision_m: 1.0496 - recall_m: 1.4107
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3743 - accuracy: 0.8533 - f1_m: 1.2014 - precision_m: 1.0489 - recall_m: 1.4116
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3745 - accuracy: 0.8533 - f1_m: 1.2001 - precision_m: 1.0467 - recall_m: 1.4113
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3724 - accuracy: 0.8541 - f1_m: 1.1978 - precision_m: 1.0448 - recall_m: 1.4087
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3731 - accuracy: 0.8541 - f1_m: 1.1975 - precision_m: 1.0452 - recall_m: 1.4075
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3711 - accuracy: 0.8544 - f1_m: 1.1948 - precision_m: 1.0419 - recall_m: 1.4058
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3702 - accuracy: 0.8546 - f1_m: 1.1983 - precision_m: 1.0457 - recall_m: 1.4089
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3702 - accuracy: 0.8540 - f1_m: 1.1966 - precision_m: 1.0443 - recall_m: 1.4073
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3686 - accuracy: 0.8559 - f1_m: 1.1949 - precision_m: 1.0413 - recall_m: 1.4073
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3704 - accuracy: 0.8549 - f1_m: 1.1948 - precision_m: 1.0423 - recall_m: 1.4055
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3687 - accuracy: 0.8554 - f1_m: 1.1964 - precision_m: 1.0432 - recall_m: 1.4084
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3665 - accuracy: 0.8560 - f1_m: 1.1927 - precision_m: 1.0386 - recall_m: 1.4060
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3659 - accuracy: 0.8564 - f1_m: 1.1945 - precision_m: 1.0410 - recall_m: 1.4065
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3657 - accuracy: 0.8559 - f1_m: 1.1933 - precision_m: 1.0404 - recall_m: 1.4047
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3654 - accuracy: 0.8562 - f1_m: 1.1932 - precision_m: 1.0397 - recall_m: 1.4054
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3654 - accuracy: 0.8560 - f1_m: 1.1919 - precision_m: 1.0380 - recall_m: 1.4048
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3632 - accuracy: 0.8565 - f1_m: 1.1886 - precision_m: 1.0347 - recall_m: 1.4021
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3628 - accuracy: 0.8570 - f1_m: 1.1923 - precision_m: 1.0390 - recall_m: 1.4046
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3629 - accuracy: 0.8570 - f1_m: 1.1916 - precision_m: 1.0377 - recall_m: 1.4047
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3621 - accuracy: 0.8574 - f1_m: 1.1908 - precision_m: 1.0376 - recall_m: 1.4029
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3623 - accuracy: 0.8573 - f1_m: 1.1876 - precision_m: 1.0334 - recall_m: 1.4013
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3604 - accuracy: 0.8577 - f1_m: 1.1892 - precision_m: 1.0358 - recall_m: 1.4017
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3613 - accuracy: 0.8576 - f1_m: 1.1896 - precision_m: 1.0351 - recall_m: 1.4035
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3599 - accuracy: 0.8575 - f1_m: 1.1902 - precision_m: 1.0359 - recall_m: 1.4042
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3596 - accuracy: 0.8578 - f1_m: 1.1871 - precision_m: 1.0333 - recall_m: 1.4003
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3585 - accuracy: 0.8581 - f1_m: 1.1881 - precision_m: 1.0338 - recall_m: 1.4025
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3583 - accuracy: 0.8583 - f1_m: 1.1886 - precision_m: 1.0342 - recall_m: 1.4028
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3574 - accuracy: 0.8589 - f1_m: 1.1834 - precision_m: 1.0292 - recall_m: 1.3976
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3575 - accuracy: 0.8588 - f1_m: 1.1873 - precision_m: 1.0332 - recall_m: 1.4011
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3563 - accuracy: 0.8589 - f1_m: 1.1854 - precision_m: 1.0308 - recall_m: 1.4003
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3571 - accuracy: 0.8590 - f1_m: 1.1866 - precision_m: 1.0323 - recall_m: 1.4008
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3549 - accuracy: 0.8595 - f1_m: 1.1849 - precision_m: 1.0306 - recall_m: 1.3994
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3558 - accuracy: 0.8594 - f1_m: 1.1858 - precision_m: 1.0314 - recall_m: 1.4001
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3556 - accuracy: 0.8595 - f1_m: 1.1835 - precision_m: 1.0291 - recall_m: 1.3977
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3542 - accuracy: 0.8595 - f1_m: 1.1809 - precision_m: 1.0254 - recall_m: 1.3974
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3540 - accuracy: 0.8600 - f1_m: 1.1833 - precision_m: 1.0289 - recall_m: 1.3977
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3525 - accuracy: 0.8609 - f1_m: 1.1845 - precision_m: 1.0303 - recall_m: 1.3986
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3531 - accuracy: 0.8602 - f1_m: 1.1812 - precision_m: 1.0267 - recall_m: 1.3959
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3533 - accuracy: 0.8607 - f1_m: 1.1831 - precision_m: 1.0288 - recall_m: 1.3975
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3516 - accuracy: 0.8609 - f1_m: 1.1813 - precision_m: 1.0265 - recall_m: 1.3967
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3524 - accuracy: 0.8601 - f1_m: 1.1821 - precision_m: 1.0274 - recall_m: 1.3976
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3510 - accuracy: 0.8609 - f1_m: 1.1805 - precision_m: 1.0256 - recall_m: 1.3959
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3510 - accuracy: 0.8616 - f1_m: 1.1816 - precision_m: 1.0261 - recall_m: 1.3980
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3497 - accuracy: 0.8616 - f1_m: 1.1824 - precision_m: 1.0279 - recall_m: 1.3971
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3502 - accuracy: 0.8610 - f1_m: 1.1798 - precision_m: 1.0245 - recall_m: 1.3959
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3495 - accuracy: 0.8623 - f1_m: 1.1814 - precision_m: 1.0265 - recall_m: 1.3968
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3493 - accuracy: 0.8620 - f1_m: 1.1809 - precision_m: 1.0260 - recall_m: 1.3966
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3490 - accuracy: 0.8621 - f1_m: 1.1804 - precision_m: 1.0253 - recall_m: 1.3964
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3483 - accuracy: 0.8617 - f1_m: 1.1814 - precision_m: 1.0265 - recall_m: 1.3972
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3484 - accuracy: 0.8622 - f1_m: 1.1793 - precision_m: 1.0243 - recall_m: 1.3952
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3470 - accuracy: 0.8624 - f1_m: 1.1788 - precision_m: 1.0235 - recall_m: 1.3949
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3483 - accuracy: 0.8627 - f1_m: 1.1784 - precision_m: 1.0231 - recall_m: 1.3949
Epoch 97/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3457 - accuracy: 0.8632 - f1_m: 1.1774 - precision_m: 1.0220 - recall_m: 1.3942
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3466 - accuracy: 0.8631 - f1_m: 1.1766 - precision_m: 1.0211 - recall_m: 1.3936
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3469 - accuracy: 0.8627 - f1_m: 1.1772 - precision_m: 1.0212 - recall_m: 1.3949
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3461 - accuracy: 0.8627 - f1_m: 1.1783 - precision_m: 1.0224 - recall_m: 1.3960
2658/2658 [==============================] - 3s 936us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4125 - accuracy: 0.8491 - f1_m: 1.1640 - precision_m: 1.0130 - recall_m: 1.3882
score on test: 0.8490850329399109
5395/5395 [==============================] - 7s 1ms/step - loss: 0.3348 - accuracy: 0.8670 - f1_m: 1.1652 - precision_m: 1.0165 - recall_m: 1.3862
score on train: 0.8670393228530884

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
from sklearn.preprocessing import LabelBinarizer

# function for scoring roc auc score for multi-class
def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
    lb = LabelBinarizer()
    lb.fit(y_test)
    y_test = lb.transform(y_test)
    y_pred = lb.transform(y_pred)

    for (idx, c_label) in enumerate(attack_cat):
        fpr, tpr, thresholds = roc_curve(y_test[:,idx].astype(int), y_pred[:,idx])
        c_ax.plot(fpr, tpr, label = '%s (AUC:%0.2f)'  % (c_label, auc(fpr, tpr)))
    c_ax.plot(fpr, fpr, 'b-', label = 'Random Guessing')
    return roc_auc_score(y_test, y_pred, average=average)


print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.7597874611698036

Noise range 1

noise2 = 2
X_train_mod2 = modify_dataset(X_train, variables_to_poison, noise2)

X_train_mod2

array([[ 1.79214449, -0.2082886 , -3.05107026, ...,  1.45449919,
        -1.42868768, -0.12070382],
       [-0.57638416,  1.78686151,  0.96697031, ..., -1.80676416,
        -1.94982839, -0.12070382],
       [-1.0569863 , -0.20828827, -3.08962164, ...,  1.63067824,
         6.19664455, -0.12070382],
       ...,
       [-0.47187522, -0.06694436, -3.69772927, ..., -1.34439782,
        -0.73376123, -0.12070382],
       [-0.1666061 , -0.18783648,  0.39679066, ...,  2.3451629 ,
        -0.88685302, -0.12070382],
       [-0.6757627 , -0.20011705,  0.5396698 , ..., -1.54171135,
        -0.79964479, -0.12070382]])

from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod2, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod2, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['2']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.891776826457167
score on train: 0.8915483983085211
              precision    recall  f1-score   support

           0       0.93      0.75      0.83     30772
           1       0.87      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.90      0.86      0.88     85028
weighted avg       0.90      0.89      0.89     85028

# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9547

rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod2, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod2, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['2']=acc

<ipython-input-28-7c4992ce0511>:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod2, y_train)

score on test: 0.9503222467892929
score on train: 0.9509818687366043
              precision    recall  f1-score   support

           0       0.95      0.91      0.93     30772
           1       0.95      0.97      0.96     54256

    accuracy                           0.95     85028
   macro avg       0.95      0.94      0.95     85028
weighted avg       0.95      0.95      0.95     85028

rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9876

xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod2, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod2, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['2']=acc

score on test: 0.9686926659453357
score on train: 0.9923420031280774
              precision    recall  f1-score   support

           0       0.97      0.95      0.96     30772
           1       0.97      0.98      0.98     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.96      0.97     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9960

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod2, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod2,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores = model.evaluate(X_test, y_test, verbose=0)
res_nn_b['2']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.2267 - accuracy: 0.8973 - f1_m: 0.9220 - precision_m: 0.8944 - recall_m: 0.9531
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1778 - accuracy: 0.9187 - f1_m: 0.9370 - precision_m: 0.9247 - recall_m: 0.9506
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1558 - accuracy: 0.9280 - f1_m: 0.9437 - precision_m: 0.9375 - recall_m: 0.9508
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1423 - accuracy: 0.9350 - f1_m: 0.9489 - precision_m: 0.9475 - recall_m: 0.9512
Epoch 5/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.1309 - accuracy: 0.9407 - f1_m: 0.9534 - precision_m: 0.9533 - recall_m: 0.9543
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1121 - accuracy: 0.9512 - f1_m: 0.9616 - precision_m: 0.9615 - recall_m: 0.9623
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0985 - accuracy: 0.9582 - f1_m: 0.9671 - precision_m: 0.9670 - recall_m: 0.9676
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0915 - accuracy: 0.9614 - f1_m: 0.9696 - precision_m: 0.9699 - recall_m: 0.9698
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0867 - accuracy: 0.9637 - f1_m: 0.9714 - precision_m: 0.9717 - recall_m: 0.9715
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0831 - accuracy: 0.9654 - f1_m: 0.9728 - precision_m: 0.9736 - recall_m: 0.9724
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0813 - accuracy: 0.9662 - f1_m: 0.9734 - precision_m: 0.9742 - recall_m: 0.9729
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0791 - accuracy: 0.9669 - f1_m: 0.9739 - precision_m: 0.9750 - recall_m: 0.9732
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0773 - accuracy: 0.9673 - f1_m: 0.9742 - precision_m: 0.9757 - recall_m: 0.9731
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0764 - accuracy: 0.9677 - f1_m: 0.9746 - precision_m: 0.9760 - recall_m: 0.9735
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0741 - accuracy: 0.9694 - f1_m: 0.9759 - precision_m: 0.9773 - recall_m: 0.9748
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0728 - accuracy: 0.9698 - f1_m: 0.9762 - precision_m: 0.9779 - recall_m: 0.9748
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0720 - accuracy: 0.9699 - f1_m: 0.9763 - precision_m: 0.9780 - recall_m: 0.9748
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0711 - accuracy: 0.9705 - f1_m: 0.9767 - precision_m: 0.9787 - recall_m: 0.9750
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0698 - accuracy: 0.9708 - f1_m: 0.9769 - precision_m: 0.9789 - recall_m: 0.9753
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0697 - accuracy: 0.9708 - f1_m: 0.9770 - precision_m: 0.9791 - recall_m: 0.9753
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0682 - accuracy: 0.9714 - f1_m: 0.9775 - precision_m: 0.9797 - recall_m: 0.9755
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0671 - accuracy: 0.9721 - f1_m: 0.9780 - precision_m: 0.9803 - recall_m: 0.9760
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0667 - accuracy: 0.9718 - f1_m: 0.9778 - precision_m: 0.9797 - recall_m: 0.9762
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0658 - accuracy: 0.9730 - f1_m: 0.9787 - precision_m: 0.9808 - recall_m: 0.9768
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0642 - accuracy: 0.9731 - f1_m: 0.9788 - precision_m: 0.9811 - recall_m: 0.9769
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0640 - accuracy: 0.9732 - f1_m: 0.9788 - precision_m: 0.9812 - recall_m: 0.9767
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0649 - accuracy: 0.9731 - f1_m: 0.9788 - precision_m: 0.9810 - recall_m: 0.9769
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0625 - accuracy: 0.9739 - f1_m: 0.9794 - precision_m: 0.9818 - recall_m: 0.9772
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0619 - accuracy: 0.9743 - f1_m: 0.9797 - precision_m: 0.9821 - recall_m: 0.9777
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0624 - accuracy: 0.9741 - f1_m: 0.9795 - precision_m: 0.9818 - recall_m: 0.9775
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0610 - accuracy: 0.9744 - f1_m: 0.9799 - precision_m: 0.9821 - recall_m: 0.9779
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0601 - accuracy: 0.9750 - f1_m: 0.9803 - precision_m: 0.9829 - recall_m: 0.9780
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0600 - accuracy: 0.9748 - f1_m: 0.9801 - precision_m: 0.9823 - recall_m: 0.9782
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0596 - accuracy: 0.9751 - f1_m: 0.9804 - precision_m: 0.9828 - recall_m: 0.9783
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0588 - accuracy: 0.9752 - f1_m: 0.9804 - precision_m: 0.9830 - recall_m: 0.9781
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0579 - accuracy: 0.9758 - f1_m: 0.9810 - precision_m: 0.9835 - recall_m: 0.9787
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0582 - accuracy: 0.9761 - f1_m: 0.9812 - precision_m: 0.9835 - recall_m: 0.9791
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0568 - accuracy: 0.9762 - f1_m: 0.9813 - precision_m: 0.9837 - recall_m: 0.9791
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0568 - accuracy: 0.9767 - f1_m: 0.9816 - precision_m: 0.9841 - recall_m: 0.9795
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0554 - accuracy: 0.9768 - f1_m: 0.9817 - precision_m: 0.9844 - recall_m: 0.9793
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0562 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9838 - recall_m: 0.9793
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0552 - accuracy: 0.9768 - f1_m: 0.9818 - precision_m: 0.9840 - recall_m: 0.9798
Epoch 43/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0547 - accuracy: 0.9770 - f1_m: 0.9819 - precision_m: 0.9844 - recall_m: 0.9796
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0537 - accuracy: 0.9778 - f1_m: 0.9825 - precision_m: 0.9848 - recall_m: 0.9804
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0540 - accuracy: 0.9776 - f1_m: 0.9823 - precision_m: 0.9849 - recall_m: 0.9800
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0526 - accuracy: 0.9779 - f1_m: 0.9826 - precision_m: 0.9850 - recall_m: 0.9805
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0528 - accuracy: 0.9781 - f1_m: 0.9827 - precision_m: 0.9851 - recall_m: 0.9806
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0524 - accuracy: 0.9783 - f1_m: 0.9829 - precision_m: 0.9851 - recall_m: 0.9810
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0518 - accuracy: 0.9784 - f1_m: 0.9830 - precision_m: 0.9852 - recall_m: 0.9811
Epoch 50/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0517 - accuracy: 0.9785 - f1_m: 0.9830 - precision_m: 0.9853 - recall_m: 0.9811
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0516 - accuracy: 0.9781 - f1_m: 0.9827 - precision_m: 0.9851 - recall_m: 0.9806
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0510 - accuracy: 0.9789 - f1_m: 0.9834 - precision_m: 0.9857 - recall_m: 0.9813
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0504 - accuracy: 0.9790 - f1_m: 0.9835 - precision_m: 0.9856 - recall_m: 0.9816
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0503 - accuracy: 0.9789 - f1_m: 0.9834 - precision_m: 0.9857 - recall_m: 0.9812
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0500 - accuracy: 0.9793 - f1_m: 0.9836 - precision_m: 0.9861 - recall_m: 0.9814
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0493 - accuracy: 0.9793 - f1_m: 0.9837 - precision_m: 0.9861 - recall_m: 0.9817
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0488 - accuracy: 0.9796 - f1_m: 0.9839 - precision_m: 0.9861 - recall_m: 0.9820
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0491 - accuracy: 0.9799 - f1_m: 0.9842 - precision_m: 0.9862 - recall_m: 0.9824
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0483 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9863 - recall_m: 0.9819
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0477 - accuracy: 0.9801 - f1_m: 0.9843 - precision_m: 0.9864 - recall_m: 0.9825
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0474 - accuracy: 0.9803 - f1_m: 0.9845 - precision_m: 0.9866 - recall_m: 0.9825
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0466 - accuracy: 0.9805 - f1_m: 0.9846 - precision_m: 0.9868 - recall_m: 0.9827
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0471 - accuracy: 0.9803 - f1_m: 0.9845 - precision_m: 0.9865 - recall_m: 0.9827
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0461 - accuracy: 0.9810 - f1_m: 0.9850 - precision_m: 0.9874 - recall_m: 0.9828
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0462 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9870 - recall_m: 0.9830
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9805 - f1_m: 0.9847 - precision_m: 0.9868 - recall_m: 0.9828
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0452 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9873 - recall_m: 0.9831
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0461 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9873 - recall_m: 0.9830
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0438 - accuracy: 0.9819 - f1_m: 0.9858 - precision_m: 0.9879 - recall_m: 0.9838
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0445 - accuracy: 0.9817 - f1_m: 0.9856 - precision_m: 0.9876 - recall_m: 0.9838
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0447 - accuracy: 0.9813 - f1_m: 0.9853 - precision_m: 0.9876 - recall_m: 0.9832
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0441 - accuracy: 0.9815 - f1_m: 0.9854 - precision_m: 0.9877 - recall_m: 0.9833
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0424 - accuracy: 0.9822 - f1_m: 0.9860 - precision_m: 0.9883 - recall_m: 0.9839
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0435 - accuracy: 0.9822 - f1_m: 0.9859 - precision_m: 0.9877 - recall_m: 0.9843
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0424 - accuracy: 0.9824 - f1_m: 0.9861 - precision_m: 0.9883 - recall_m: 0.9842
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0435 - accuracy: 0.9822 - f1_m: 0.9860 - precision_m: 0.9882 - recall_m: 0.9840
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0422 - accuracy: 0.9826 - f1_m: 0.9863 - precision_m: 0.9884 - recall_m: 0.9843
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0415 - accuracy: 0.9829 - f1_m: 0.9865 - precision_m: 0.9885 - recall_m: 0.9847
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0413 - accuracy: 0.9829 - f1_m: 0.9865 - precision_m: 0.9885 - recall_m: 0.9847
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0411 - accuracy: 0.9833 - f1_m: 0.9868 - precision_m: 0.9888 - recall_m: 0.9850
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0411 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9890 - recall_m: 0.9850
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0407 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9889 - recall_m: 0.9850
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0409 - accuracy: 0.9830 - f1_m: 0.9866 - precision_m: 0.9887 - recall_m: 0.9847
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0397 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9890 - recall_m: 0.9849
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0394 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9890 - recall_m: 0.9850
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0393 - accuracy: 0.9839 - f1_m: 0.9873 - precision_m: 0.9893 - recall_m: 0.9855
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0397 - accuracy: 0.9834 - f1_m: 0.9869 - precision_m: 0.9890 - recall_m: 0.9851
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0408 - accuracy: 0.9836 - f1_m: 0.9871 - precision_m: 0.9890 - recall_m: 0.9853
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0389 - accuracy: 0.9837 - f1_m: 0.9872 - precision_m: 0.9891 - recall_m: 0.9854
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0376 - accuracy: 0.9846 - f1_m: 0.9879 - precision_m: 0.9898 - recall_m: 0.9861
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0385 - accuracy: 0.9838 - f1_m: 0.9873 - precision_m: 0.9891 - recall_m: 0.9856
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0379 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9895 - recall_m: 0.9858
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0383 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9895 - recall_m: 0.9857
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0372 - accuracy: 0.9847 - f1_m: 0.9880 - precision_m: 0.9900 - recall_m: 0.9861
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0381 - accuracy: 0.9843 - f1_m: 0.9876 - precision_m: 0.9894 - recall_m: 0.9860
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0374 - accuracy: 0.9848 - f1_m: 0.9880 - precision_m: 0.9900 - recall_m: 0.9862
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0368 - accuracy: 0.9849 - f1_m: 0.9881 - precision_m: 0.9900 - recall_m: 0.9864
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0379 - accuracy: 0.9844 - f1_m: 0.9877 - precision_m: 0.9893 - recall_m: 0.9863
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0363 - accuracy: 0.9851 - f1_m: 0.9883 - precision_m: 0.9901 - recall_m: 0.9866
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0371 - accuracy: 0.9846 - f1_m: 0.9879 - precision_m: 0.9899 - recall_m: 0.9860
2658/2658 [==============================] - 3s 939us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0822 - accuracy: 0.9730 - f1_m: 0.9783 - precision_m: 0.9840 - recall_m: 0.9737
score on test: 0.9730324149131775
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0346 - accuracy: 0.9867 - f1_m: 0.9893 - precision_m: 0.9929 - recall_m: 0.9863
score on train: 0.9866709113121033

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod2, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod2, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['2']=acc

score on test: 0.8045114550501011
score on train: 0.8270752476394602
              precision    recall  f1-score   support

           0       0.90      0.93      0.92     30772
           1       0.96      0.96      0.96     19423
           2       0.60      0.82      0.69     14709
           3       0.68      0.59      0.63      7921
           4       0.42      0.15      0.22      5357
           5       0.72      0.72      0.72      4578
           6       0.65      0.02      0.04       909
           7       0.16      0.02      0.03       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.80     85028
   macro avg       0.51      0.42      0.42     85028
weighted avg       0.79      0.80      0.78     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(dtree1, X_train_mod2, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod2, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod2, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['2']=acc

<ipython-input-36-63dc6513c11b>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod2, y_multi_train)

score on test: 0.7980900409276944
score on train: 0.815958987429763
              precision    recall  f1-score   support

           0       0.84      0.96      0.90     30772
           1       1.00      0.97      0.98     19423
           2       0.57      0.89      0.69     14709
           3       0.75      0.44      0.55      7921
           4       0.45      0.01      0.03      5357
           5       0.90      0.63      0.74      4578
           6       0.00      0.00      0.00       909
           7       1.00      0.00      0.01       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.80     85028
   macro avg       0.55      0.39      0.39     85028
weighted avg       0.79      0.80      0.76     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod2, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod2, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod2, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['2']=acc

score on test: 0.8517194335983441
score on train: 0.9090772171696693
              precision    recall  f1-score   support

           0       0.95      0.98      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.73      0.71      0.72     14709
           3       0.79      0.79      0.79      7921
           4       0.38      0.52      0.44      5357
           5       0.90      0.76      0.82      4578
           6       0.63      0.11      0.19       909
           7       0.51      0.08      0.14       780
           8       0.62      0.38      0.47       513
           9       0.70      0.11      0.18        66

    accuracy                           0.85     85028
   macro avg       0.72      0.54      0.57     85028
weighted avg       0.86      0.85      0.85     85028

plot_ROC_curve(xgb2, X_train_mod2, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod2,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
scores=model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['2']=scores[1]
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod2,y_multi_train)[1]))
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_12 (Dense)            (None, 128)               5504      
                                                                 
 dense_13 (Dense)            (None, 64)                8256      
                                                                 
 dense_14 (Dense)            (None, 64)                4160      
                                                                 
 dense_15 (Dense)            (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.8160 - accuracy: 0.7022 - f1_m: 1.6790 - precision_m: 6234247.0000 - recall_m: 1.6774
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6323 - accuracy: 0.7648 - f1_m: 1.4841 - precision_m: 1.3890 - recall_m: 1.6025
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5773 - accuracy: 0.7847 - f1_m: 1.4074 - precision_m: 1.2883 - recall_m: 1.5583
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5463 - accuracy: 0.7942 - f1_m: 1.3593 - precision_m: 1.2322 - recall_m: 1.5229
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5285 - accuracy: 0.8002 - f1_m: 1.3384 - precision_m: 1.2070 - recall_m: 1.5087
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5141 - accuracy: 0.8053 - f1_m: 1.3175 - precision_m: 1.1821 - recall_m: 1.4943
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5022 - accuracy: 0.8097 - f1_m: 1.3033 - precision_m: 1.1652 - recall_m: 1.4848
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4882 - accuracy: 0.8153 - f1_m: 1.2896 - precision_m: 1.1487 - recall_m: 1.4760
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4722 - accuracy: 0.8223 - f1_m: 1.2724 - precision_m: 1.1296 - recall_m: 1.4626
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4592 - accuracy: 0.8272 - f1_m: 1.2596 - precision_m: 1.1158 - recall_m: 1.4517
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4517 - accuracy: 0.8292 - f1_m: 1.2536 - precision_m: 1.1101 - recall_m: 1.4463
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4439 - accuracy: 0.8319 - f1_m: 1.2471 - precision_m: 1.1017 - recall_m: 1.4431
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4398 - accuracy: 0.8336 - f1_m: 1.2463 - precision_m: 1.1009 - recall_m: 1.4422
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4361 - accuracy: 0.8344 - f1_m: 1.2411 - precision_m: 1.0952 - recall_m: 1.4382
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4322 - accuracy: 0.8350 - f1_m: 1.2392 - precision_m: 1.0927 - recall_m: 1.4371
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4281 - accuracy: 0.8361 - f1_m: 1.2350 - precision_m: 1.0878 - recall_m: 1.4344
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4259 - accuracy: 0.8371 - f1_m: 1.2340 - precision_m: 1.0869 - recall_m: 1.4332
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4223 - accuracy: 0.8381 - f1_m: 1.2313 - precision_m: 1.0841 - recall_m: 1.4307
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4187 - accuracy: 0.8394 - f1_m: 1.2321 - precision_m: 1.0843 - recall_m: 1.4330
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4174 - accuracy: 0.8398 - f1_m: 1.2298 - precision_m: 1.0812 - recall_m: 1.4319
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4138 - accuracy: 0.8409 - f1_m: 1.2278 - precision_m: 1.0797 - recall_m: 1.4294
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4136 - accuracy: 0.8416 - f1_m: 1.2258 - precision_m: 1.0772 - recall_m: 1.4280
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4109 - accuracy: 0.8414 - f1_m: 1.2235 - precision_m: 1.0742 - recall_m: 1.4269
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4082 - accuracy: 0.8425 - f1_m: 1.2246 - precision_m: 1.0761 - recall_m: 1.4266
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4065 - accuracy: 0.8434 - f1_m: 1.2232 - precision_m: 1.0744 - recall_m: 1.4259
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4053 - accuracy: 0.8436 - f1_m: 1.2226 - precision_m: 1.0736 - recall_m: 1.4256
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4027 - accuracy: 0.8447 - f1_m: 1.2202 - precision_m: 1.0709 - recall_m: 1.4237
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4023 - accuracy: 0.8446 - f1_m: 1.2183 - precision_m: 1.0684 - recall_m: 1.4230
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4009 - accuracy: 0.8449 - f1_m: 1.2179 - precision_m: 1.0686 - recall_m: 1.4210
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3997 - accuracy: 0.8459 - f1_m: 1.2187 - precision_m: 1.0695 - recall_m: 1.4220
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3973 - accuracy: 0.8468 - f1_m: 1.2156 - precision_m: 1.0653 - recall_m: 1.4212
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3969 - accuracy: 0.8466 - f1_m: 1.2151 - precision_m: 1.0652 - recall_m: 1.4196
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3957 - accuracy: 0.8477 - f1_m: 1.2130 - precision_m: 1.0625 - recall_m: 1.4189
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3939 - accuracy: 0.8475 - f1_m: 1.2130 - precision_m: 1.0621 - recall_m: 1.4198
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3924 - accuracy: 0.8480 - f1_m: 1.2131 - precision_m: 1.0624 - recall_m: 1.4196
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3926 - accuracy: 0.8482 - f1_m: 1.2113 - precision_m: 1.0598 - recall_m: 1.4190
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3896 - accuracy: 0.8483 - f1_m: 1.2107 - precision_m: 1.0597 - recall_m: 1.4177
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3898 - accuracy: 0.8488 - f1_m: 1.2080 - precision_m: 1.0561 - recall_m: 1.4164
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3881 - accuracy: 0.8493 - f1_m: 1.2100 - precision_m: 1.0591 - recall_m: 1.4169
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3869 - accuracy: 0.8495 - f1_m: 1.2084 - precision_m: 1.0571 - recall_m: 1.4162
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3853 - accuracy: 0.8498 - f1_m: 1.2088 - precision_m: 1.0571 - recall_m: 1.4169
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3852 - accuracy: 0.8501 - f1_m: 1.2070 - precision_m: 1.0558 - recall_m: 1.4149
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3841 - accuracy: 0.8502 - f1_m: 1.2048 - precision_m: 1.0538 - recall_m: 1.4124
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3834 - accuracy: 0.8512 - f1_m: 1.2071 - precision_m: 1.0556 - recall_m: 1.4151
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3826 - accuracy: 0.8516 - f1_m: 1.2041 - precision_m: 1.0521 - recall_m: 1.4133
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3811 - accuracy: 0.8513 - f1_m: 1.2054 - precision_m: 1.0536 - recall_m: 1.4139
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3797 - accuracy: 0.8523 - f1_m: 1.2033 - precision_m: 1.0515 - recall_m: 1.4120
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3796 - accuracy: 0.8521 - f1_m: 1.2025 - precision_m: 1.0507 - recall_m: 1.4116
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3780 - accuracy: 0.8526 - f1_m: 1.2024 - precision_m: 1.0508 - recall_m: 1.4109
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3772 - accuracy: 0.8526 - f1_m: 1.2015 - precision_m: 1.0494 - recall_m: 1.4113
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3777 - accuracy: 0.8532 - f1_m: 1.1988 - precision_m: 1.0465 - recall_m: 1.4091
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3759 - accuracy: 0.8541 - f1_m: 1.1987 - precision_m: 1.0465 - recall_m: 1.4091
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3763 - accuracy: 0.8536 - f1_m: 1.1985 - precision_m: 1.0460 - recall_m: 1.4088
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3740 - accuracy: 0.8547 - f1_m: 1.1999 - precision_m: 1.0477 - recall_m: 1.4094
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3742 - accuracy: 0.8547 - f1_m: 1.1978 - precision_m: 1.0451 - recall_m: 1.4084
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3728 - accuracy: 0.8546 - f1_m: 1.1966 - precision_m: 1.0433 - recall_m: 1.4077
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3725 - accuracy: 0.8547 - f1_m: 1.1981 - precision_m: 1.0463 - recall_m: 1.4075
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3723 - accuracy: 0.8542 - f1_m: 1.1935 - precision_m: 1.0403 - recall_m: 1.4053
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3710 - accuracy: 0.8554 - f1_m: 1.1950 - precision_m: 1.0422 - recall_m: 1.4058
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3708 - accuracy: 0.8558 - f1_m: 1.1945 - precision_m: 1.0418 - recall_m: 1.4052
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3700 - accuracy: 0.8553 - f1_m: 1.1954 - precision_m: 1.0433 - recall_m: 1.4057
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3688 - accuracy: 0.8559 - f1_m: 1.1908 - precision_m: 1.0377 - recall_m: 1.4027
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3682 - accuracy: 0.8555 - f1_m: 1.1944 - precision_m: 1.0415 - recall_m: 1.4055
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3676 - accuracy: 0.8564 - f1_m: 1.1938 - precision_m: 1.0414 - recall_m: 1.4046
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3670 - accuracy: 0.8562 - f1_m: 1.1934 - precision_m: 1.0404 - recall_m: 1.4050
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3666 - accuracy: 0.8566 - f1_m: 1.1907 - precision_m: 1.0374 - recall_m: 1.4030
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3656 - accuracy: 0.8577 - f1_m: 1.1912 - precision_m: 1.0381 - recall_m: 1.4031
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3658 - accuracy: 0.8569 - f1_m: 1.1899 - precision_m: 1.0372 - recall_m: 1.4012
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3648 - accuracy: 0.8573 - f1_m: 1.1896 - precision_m: 1.0361 - recall_m: 1.4020
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3663 - accuracy: 0.8566 - f1_m: 1.1910 - precision_m: 1.0376 - recall_m: 1.4033
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3644 - accuracy: 0.8573 - f1_m: 1.1900 - precision_m: 1.0362 - recall_m: 1.4028
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3636 - accuracy: 0.8575 - f1_m: 1.1895 - precision_m: 1.0357 - recall_m: 1.4024
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3630 - accuracy: 0.8574 - f1_m: 1.1904 - precision_m: 1.0374 - recall_m: 1.4021
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3612 - accuracy: 0.8583 - f1_m: 1.1896 - precision_m: 1.0362 - recall_m: 1.4020
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3630 - accuracy: 0.8571 - f1_m: 1.1856 - precision_m: 1.0318 - recall_m: 1.3991
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3623 - accuracy: 0.8586 - f1_m: 1.1862 - precision_m: 1.0329 - recall_m: 1.3986
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3608 - accuracy: 0.8582 - f1_m: 1.1876 - precision_m: 1.0340 - recall_m: 1.4003
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3599 - accuracy: 0.8589 - f1_m: 1.1889 - precision_m: 1.0355 - recall_m: 1.4013
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3599 - accuracy: 0.8589 - f1_m: 1.1853 - precision_m: 1.0315 - recall_m: 1.3982
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3603 - accuracy: 0.8586 - f1_m: 1.1868 - precision_m: 1.0337 - recall_m: 1.3988
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3581 - accuracy: 0.8587 - f1_m: 1.1846 - precision_m: 1.0305 - recall_m: 1.3985
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3580 - accuracy: 0.8593 - f1_m: 1.1859 - precision_m: 1.0324 - recall_m: 1.3992
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3576 - accuracy: 0.8595 - f1_m: 1.1861 - precision_m: 1.0328 - recall_m: 1.3989
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3580 - accuracy: 0.8591 - f1_m: 1.1866 - precision_m: 1.0329 - recall_m: 1.3993
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3573 - accuracy: 0.8594 - f1_m: 1.1828 - precision_m: 1.0285 - recall_m: 1.3970
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3574 - accuracy: 0.8588 - f1_m: 1.1837 - precision_m: 1.0292 - recall_m: 1.3982
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3572 - accuracy: 0.8598 - f1_m: 1.1840 - precision_m: 1.0298 - recall_m: 1.3981
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3555 - accuracy: 0.8605 - f1_m: 1.1868 - precision_m: 1.0333 - recall_m: 1.3995
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3554 - accuracy: 0.8591 - f1_m: 1.1847 - precision_m: 1.0309 - recall_m: 1.3980
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3547 - accuracy: 0.8607 - f1_m: 1.1836 - precision_m: 1.0293 - recall_m: 1.3977
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3546 - accuracy: 0.8607 - f1_m: 1.1836 - precision_m: 1.0295 - recall_m: 1.3973
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3540 - accuracy: 0.8606 - f1_m: 1.1818 - precision_m: 1.0272 - recall_m: 1.3964
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3537 - accuracy: 0.8603 - f1_m: 1.1843 - precision_m: 1.0301 - recall_m: 1.3979
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3535 - accuracy: 0.8602 - f1_m: 1.1835 - precision_m: 1.0295 - recall_m: 1.3972
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3529 - accuracy: 0.8603 - f1_m: 1.1829 - precision_m: 1.0286 - recall_m: 1.3971
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3522 - accuracy: 0.8611 - f1_m: 1.1805 - precision_m: 1.0259 - recall_m: 1.3955
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3526 - accuracy: 0.8607 - f1_m: 1.1811 - precision_m: 1.0271 - recall_m: 1.3949
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3522 - accuracy: 0.8602 - f1_m: 1.1825 - precision_m: 1.0275 - recall_m: 1.3980
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3526 - accuracy: 0.8615 - f1_m: 1.1812 - precision_m: 1.0262 - recall_m: 1.3964
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3509 - accuracy: 0.8618 - f1_m: 1.1798 - precision_m: 1.0252 - recall_m: 1.3945
2658/2658 [==============================] - 2s 892us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4144 - accuracy: 0.8478 - f1_m: 1.2164 - precision_m: 1.0758 - recall_m: 1.4217
score on test: 0.8478148579597473
5395/5395 [==============================] - 6s 1ms/step - loss: 0.3427 - accuracy: 0.8643 - f1_m: 1.2059 - precision_m: 1.0659 - recall_m: 1.4114
score on train: 0.864287793636322

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.766777028566272

Noise range 1,5

noise3 = 3
X_train_mod3 = modify_dataset(X_train, variables_to_poison, noise3)

X_train_mod3

array([[ 1.79214449, -0.2082886 ,  2.4571585 , ..., -6.83623119,
        -1.18443845, -0.12070382],
       [-0.57638416,  1.78686151,  1.12655572, ...,  1.79987319,
        -8.5542165 , -0.12070382],
       [-1.0569863 , -0.20828827, -1.44776485, ...,  1.98108405,
         7.022288  , -0.12070382],
       ...,
       [-0.47187522, -0.06694436, -0.31449456, ..., -3.62903686,
         3.60652317, -0.12070382],
       [-0.1666061 , -0.18783648, -5.31632395, ...,  2.40447373,
         3.9506843 , -0.12070382],
       [-0.6757627 , -0.20011705, -0.42302625, ..., -2.41982954,
         4.91014323, -0.12070382]])

from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod3, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod3, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['3']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.8926353671731665
score on train: 0.8934773793662747
              precision    recall  f1-score   support

           0       0.93      0.76      0.84     30772
           1       0.88      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.90      0.86      0.88     85028
weighted avg       0.90      0.89      0.89     85028

# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9540

rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod3, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod3, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['3']=acc

<ipython-input-45-1b3f14efa2b7>:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod3, y_train)

score on test: 0.9538269746436469
score on train: 0.9503157041070498
              precision    recall  f1-score   support

           0       0.96      0.91      0.93     30772
           1       0.95      0.98      0.96     54256

    accuracy                           0.95     85028
   macro avg       0.95      0.95      0.95     85028
weighted avg       0.95      0.95      0.95     85028

rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9885

xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod3, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod3, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['3']=acc

score on test: 0.968904360916404
score on train: 0.992191391994439
              precision    recall  f1-score   support

           0       0.97      0.95      0.96     30772
           1       0.97      0.98      0.98     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.96      0.97     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9961

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod3, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod3,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores=model.evaluate(X_test, y_test, verbose=0)
res_nn_b['3']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.2416 - accuracy: 0.8909 - f1_m: 0.9175 - precision_m: 0.8866 - recall_m: 0.9522
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1876 - accuracy: 0.9142 - f1_m: 0.9337 - precision_m: 0.9176 - recall_m: 0.9516
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1691 - accuracy: 0.9213 - f1_m: 0.9389 - precision_m: 0.9280 - recall_m: 0.9510
Epoch 4/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.1527 - accuracy: 0.9287 - f1_m: 0.9442 - precision_m: 0.9390 - recall_m: 0.9504
Epoch 5/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.1374 - accuracy: 0.9377 - f1_m: 0.9511 - precision_m: 0.9490 - recall_m: 0.9540
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1168 - accuracy: 0.9497 - f1_m: 0.9605 - precision_m: 0.9598 - recall_m: 0.9617
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1034 - accuracy: 0.9563 - f1_m: 0.9656 - precision_m: 0.9653 - recall_m: 0.9665
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0957 - accuracy: 0.9598 - f1_m: 0.9684 - precision_m: 0.9687 - recall_m: 0.9685
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0908 - accuracy: 0.9617 - f1_m: 0.9699 - precision_m: 0.9707 - recall_m: 0.9695
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0874 - accuracy: 0.9629 - f1_m: 0.9708 - precision_m: 0.9714 - recall_m: 0.9707
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0848 - accuracy: 0.9642 - f1_m: 0.9718 - precision_m: 0.9730 - recall_m: 0.9711
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0824 - accuracy: 0.9655 - f1_m: 0.9729 - precision_m: 0.9743 - recall_m: 0.9719
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0809 - accuracy: 0.9663 - f1_m: 0.9734 - precision_m: 0.9750 - recall_m: 0.9723
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0792 - accuracy: 0.9667 - f1_m: 0.9737 - precision_m: 0.9753 - recall_m: 0.9725
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0766 - accuracy: 0.9673 - f1_m: 0.9742 - precision_m: 0.9762 - recall_m: 0.9725
Epoch 16/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0767 - accuracy: 0.9682 - f1_m: 0.9750 - precision_m: 0.9772 - recall_m: 0.9731
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0751 - accuracy: 0.9687 - f1_m: 0.9753 - precision_m: 0.9771 - recall_m: 0.9739
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0741 - accuracy: 0.9688 - f1_m: 0.9754 - precision_m: 0.9776 - recall_m: 0.9736
Epoch 19/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0724 - accuracy: 0.9693 - f1_m: 0.9758 - precision_m: 0.9781 - recall_m: 0.9739
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0722 - accuracy: 0.9698 - f1_m: 0.9762 - precision_m: 0.9786 - recall_m: 0.9741
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0716 - accuracy: 0.9698 - f1_m: 0.9762 - precision_m: 0.9789 - recall_m: 0.9738
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0700 - accuracy: 0.9704 - f1_m: 0.9766 - precision_m: 0.9791 - recall_m: 0.9745
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0707 - accuracy: 0.9708 - f1_m: 0.9770 - precision_m: 0.9797 - recall_m: 0.9746
Epoch 24/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0682 - accuracy: 0.9710 - f1_m: 0.9771 - precision_m: 0.9794 - recall_m: 0.9751
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0683 - accuracy: 0.9711 - f1_m: 0.9772 - precision_m: 0.9799 - recall_m: 0.9749
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0667 - accuracy: 0.9720 - f1_m: 0.9779 - precision_m: 0.9806 - recall_m: 0.9756
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0657 - accuracy: 0.9724 - f1_m: 0.9783 - precision_m: 0.9809 - recall_m: 0.9759
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0656 - accuracy: 0.9725 - f1_m: 0.9783 - precision_m: 0.9812 - recall_m: 0.9758
Epoch 29/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0648 - accuracy: 0.9724 - f1_m: 0.9782 - precision_m: 0.9807 - recall_m: 0.9760
Epoch 30/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0640 - accuracy: 0.9731 - f1_m: 0.9788 - precision_m: 0.9812 - recall_m: 0.9766
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0633 - accuracy: 0.9730 - f1_m: 0.9787 - precision_m: 0.9812 - recall_m: 0.9765
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0630 - accuracy: 0.9737 - f1_m: 0.9793 - precision_m: 0.9817 - recall_m: 0.9771
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0625 - accuracy: 0.9735 - f1_m: 0.9791 - precision_m: 0.9819 - recall_m: 0.9766
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0623 - accuracy: 0.9739 - f1_m: 0.9794 - precision_m: 0.9821 - recall_m: 0.9771
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0612 - accuracy: 0.9742 - f1_m: 0.9797 - precision_m: 0.9826 - recall_m: 0.9770
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0609 - accuracy: 0.9747 - f1_m: 0.9800 - precision_m: 0.9825 - recall_m: 0.9778
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0601 - accuracy: 0.9744 - f1_m: 0.9799 - precision_m: 0.9826 - recall_m: 0.9774
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0592 - accuracy: 0.9750 - f1_m: 0.9803 - precision_m: 0.9829 - recall_m: 0.9780
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0593 - accuracy: 0.9751 - f1_m: 0.9804 - precision_m: 0.9830 - recall_m: 0.9781
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0580 - accuracy: 0.9756 - f1_m: 0.9808 - precision_m: 0.9833 - recall_m: 0.9785
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0578 - accuracy: 0.9757 - f1_m: 0.9808 - precision_m: 0.9836 - recall_m: 0.9782
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0570 - accuracy: 0.9759 - f1_m: 0.9809 - precision_m: 0.9838 - recall_m: 0.9784
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0570 - accuracy: 0.9761 - f1_m: 0.9812 - precision_m: 0.9834 - recall_m: 0.9792
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0564 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9842 - recall_m: 0.9788
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0562 - accuracy: 0.9765 - f1_m: 0.9815 - precision_m: 0.9843 - recall_m: 0.9789
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0553 - accuracy: 0.9769 - f1_m: 0.9818 - precision_m: 0.9845 - recall_m: 0.9794
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0550 - accuracy: 0.9770 - f1_m: 0.9819 - precision_m: 0.9845 - recall_m: 0.9795
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0549 - accuracy: 0.9771 - f1_m: 0.9819 - precision_m: 0.9847 - recall_m: 0.9794
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0550 - accuracy: 0.9769 - f1_m: 0.9818 - precision_m: 0.9844 - recall_m: 0.9795
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0527 - accuracy: 0.9783 - f1_m: 0.9829 - precision_m: 0.9853 - recall_m: 0.9807
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0528 - accuracy: 0.9779 - f1_m: 0.9826 - precision_m: 0.9851 - recall_m: 0.9804
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0529 - accuracy: 0.9775 - f1_m: 0.9822 - precision_m: 0.9847 - recall_m: 0.9800
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0515 - accuracy: 0.9787 - f1_m: 0.9832 - precision_m: 0.9858 - recall_m: 0.9808
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0518 - accuracy: 0.9783 - f1_m: 0.9829 - precision_m: 0.9857 - recall_m: 0.9803
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0510 - accuracy: 0.9791 - f1_m: 0.9835 - precision_m: 0.9860 - recall_m: 0.9812
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0506 - accuracy: 0.9788 - f1_m: 0.9833 - precision_m: 0.9859 - recall_m: 0.9809
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0506 - accuracy: 0.9787 - f1_m: 0.9832 - precision_m: 0.9857 - recall_m: 0.9809
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0505 - accuracy: 0.9786 - f1_m: 0.9832 - precision_m: 0.9855 - recall_m: 0.9811
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0502 - accuracy: 0.9786 - f1_m: 0.9831 - precision_m: 0.9855 - recall_m: 0.9809
Epoch 60/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0491 - accuracy: 0.9798 - f1_m: 0.9841 - precision_m: 0.9864 - recall_m: 0.9820
Epoch 61/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0493 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9864 - recall_m: 0.9818
Epoch 62/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0488 - accuracy: 0.9796 - f1_m: 0.9839 - precision_m: 0.9865 - recall_m: 0.9815
Epoch 63/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0491 - accuracy: 0.9793 - f1_m: 0.9837 - precision_m: 0.9860 - recall_m: 0.9815
Epoch 64/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0483 - accuracy: 0.9801 - f1_m: 0.9843 - precision_m: 0.9870 - recall_m: 0.9820
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0473 - accuracy: 0.9801 - f1_m: 0.9844 - precision_m: 0.9868 - recall_m: 0.9822
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0479 - accuracy: 0.9801 - f1_m: 0.9843 - precision_m: 0.9867 - recall_m: 0.9821
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0469 - accuracy: 0.9804 - f1_m: 0.9846 - precision_m: 0.9870 - recall_m: 0.9824
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0469 - accuracy: 0.9805 - f1_m: 0.9846 - precision_m: 0.9869 - recall_m: 0.9826
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0471 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9872 - recall_m: 0.9828
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0457 - accuracy: 0.9810 - f1_m: 0.9850 - precision_m: 0.9872 - recall_m: 0.9830
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0453 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9874 - recall_m: 0.9831
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0456 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9874 - recall_m: 0.9830
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0449 - accuracy: 0.9817 - f1_m: 0.9856 - precision_m: 0.9880 - recall_m: 0.9834
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9874 - recall_m: 0.9830
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0446 - accuracy: 0.9814 - f1_m: 0.9854 - precision_m: 0.9876 - recall_m: 0.9833
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0439 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9879 - recall_m: 0.9839
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0440 - accuracy: 0.9819 - f1_m: 0.9858 - precision_m: 0.9880 - recall_m: 0.9837
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0441 - accuracy: 0.9816 - f1_m: 0.9855 - precision_m: 0.9876 - recall_m: 0.9836
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0425 - accuracy: 0.9825 - f1_m: 0.9862 - precision_m: 0.9885 - recall_m: 0.9842
Epoch 80/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0431 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9879 - recall_m: 0.9840
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0436 - accuracy: 0.9821 - f1_m: 0.9859 - precision_m: 0.9880 - recall_m: 0.9841
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0431 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9880 - recall_m: 0.9838
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0421 - accuracy: 0.9827 - f1_m: 0.9864 - precision_m: 0.9885 - recall_m: 0.9844
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0421 - accuracy: 0.9828 - f1_m: 0.9865 - precision_m: 0.9886 - recall_m: 0.9845
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0415 - accuracy: 0.9827 - f1_m: 0.9864 - precision_m: 0.9882 - recall_m: 0.9847
Epoch 86/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0415 - accuracy: 0.9830 - f1_m: 0.9866 - precision_m: 0.9888 - recall_m: 0.9846
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0407 - accuracy: 0.9832 - f1_m: 0.9867 - precision_m: 0.9888 - recall_m: 0.9848
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0414 - accuracy: 0.9831 - f1_m: 0.9867 - precision_m: 0.9886 - recall_m: 0.9849
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0397 - accuracy: 0.9838 - f1_m: 0.9872 - precision_m: 0.9894 - recall_m: 0.9853
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0406 - accuracy: 0.9832 - f1_m: 0.9868 - precision_m: 0.9888 - recall_m: 0.9850
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0402 - accuracy: 0.9837 - f1_m: 0.9871 - precision_m: 0.9889 - recall_m: 0.9855
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0405 - accuracy: 0.9833 - f1_m: 0.9868 - precision_m: 0.9887 - recall_m: 0.9852
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0395 - accuracy: 0.9840 - f1_m: 0.9874 - precision_m: 0.9892 - recall_m: 0.9857
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0396 - accuracy: 0.9838 - f1_m: 0.9872 - precision_m: 0.9892 - recall_m: 0.9854
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0397 - accuracy: 0.9838 - f1_m: 0.9872 - precision_m: 0.9891 - recall_m: 0.9855
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0391 - accuracy: 0.9839 - f1_m: 0.9873 - precision_m: 0.9893 - recall_m: 0.9855
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0387 - accuracy: 0.9842 - f1_m: 0.9875 - precision_m: 0.9895 - recall_m: 0.9858
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0384 - accuracy: 0.9841 - f1_m: 0.9875 - precision_m: 0.9893 - recall_m: 0.9858
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0387 - accuracy: 0.9844 - f1_m: 0.9877 - precision_m: 0.9896 - recall_m: 0.9860
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0390 - accuracy: 0.9841 - f1_m: 0.9875 - precision_m: 0.9893 - recall_m: 0.9858
2658/2658 [==============================] - 2s 916us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0812 - accuracy: 0.9719 - f1_m: 0.9775 - precision_m: 0.9799 - recall_m: 0.9761
score on test: 0.9719151258468628
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0357 - accuracy: 0.9854 - f1_m: 0.9883 - precision_m: 0.9897 - recall_m: 0.9876
score on train: 0.9854312539100647

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod3, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod3, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['3']=acc

score on test: 0.805028931646046
score on train: 0.82693622197764

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

              precision    recall  f1-score   support

           0       0.90      0.94      0.92     30772
           1       0.97      0.97      0.97     19423
           2       0.60      0.81      0.69     14709
           3       0.69      0.59      0.63      7921
           4       0.42      0.17      0.24      5357
           5       0.70      0.72      0.71      4578
           6       0.67      0.02      0.04       909
           7       0.16      0.02      0.03       780
           8       0.43      0.01      0.01       513
           9       0.00      0.00      0.00        66

    accuracy                           0.81     85028
   macro avg       0.55      0.42      0.42     85028
weighted avg       0.79      0.81      0.79     85028

plot_ROC_curve(dtree1, X_train_mod3, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod3, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod3, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['3']=acc

<ipython-input-53-c801b2656da6>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod3, y_multi_train)

score on test: 0.7935268382179987
score on train: 0.8144528760933789
              precision    recall  f1-score   support

           0       0.82      0.97      0.89     30772
           1       1.00      0.97      0.98     19423
           2       0.57      0.89      0.69     14709
           3       0.77      0.41      0.53      7921
           4       0.84      0.01      0.02      5357
           5       0.96      0.54      0.69      4578
           6       1.00      0.00      0.00       909
           7       0.00      0.00      0.00       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.79     85028
   macro avg       0.60      0.38      0.38     85028
weighted avg       0.81      0.79      0.75     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod3, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod3, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod3, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['3']=acc

score on test: 0.8516488686079879
score on train: 0.9072640908300991
              precision    recall  f1-score   support

           0       0.95      0.97      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.72      0.72      0.72     14709
           3       0.78      0.79      0.79      7921
           4       0.38      0.49      0.43      5357
           5       0.84      0.78      0.81      4578
           6       0.65      0.10      0.17       909
           7       0.57      0.09      0.15       780
           8       0.73      0.30      0.42       513
           9       0.64      0.27      0.38        66

    accuracy                           0.85     85028
   macro avg       0.73      0.55      0.58     85028
weighted avg       0.86      0.85      0.85     85028

plot_ROC_curve(xgb2, X_train_mod3, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod3,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
scores=model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['3']=scores[1]
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod3,y_multi_train)[1]))
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_5"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_20 (Dense)            (None, 128)               5504      
                                                                 
 dense_21 (Dense)            (None, 64)                8256      
                                                                 
 dense_22 (Dense)            (None, 64)                4160      
                                                                 
 dense_23 (Dense)            (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.8478 - accuracy: 0.6924 - f1_m: 1.6986 - precision_m: 1.8692 - recall_m: 1.6835
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6573 - accuracy: 0.7569 - f1_m: 1.5087 - precision_m: 1.4206 - recall_m: 1.6169
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5967 - accuracy: 0.7761 - f1_m: 1.4310 - precision_m: 1.3171 - recall_m: 1.5740
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5616 - accuracy: 0.7883 - f1_m: 1.3826 - precision_m: 1.2591 - recall_m: 1.5398
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5388 - accuracy: 0.7962 - f1_m: 1.3509 - precision_m: 1.2231 - recall_m: 1.5157
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5231 - accuracy: 0.8012 - f1_m: 1.3281 - precision_m: 1.1945 - recall_m: 1.5020
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5105 - accuracy: 0.8055 - f1_m: 1.3158 - precision_m: 1.1810 - recall_m: 1.4921
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5018 - accuracy: 0.8092 - f1_m: 1.3029 - precision_m: 1.1644 - recall_m: 1.4851
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4936 - accuracy: 0.8116 - f1_m: 1.2992 - precision_m: 1.1617 - recall_m: 1.4799
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4858 - accuracy: 0.8143 - f1_m: 1.2896 - precision_m: 1.1495 - recall_m: 1.4750
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4747 - accuracy: 0.8187 - f1_m: 1.2779 - precision_m: 1.1346 - recall_m: 1.4691
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4660 - accuracy: 0.8223 - f1_m: 1.2691 - precision_m: 1.1249 - recall_m: 1.4621
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4570 - accuracy: 0.8250 - f1_m: 1.2629 - precision_m: 1.1182 - recall_m: 1.4575
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4488 - accuracy: 0.8284 - f1_m: 1.2580 - precision_m: 1.1126 - recall_m: 1.4539
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4438 - accuracy: 0.8309 - f1_m: 1.2514 - precision_m: 1.1045 - recall_m: 1.4493
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4386 - accuracy: 0.8327 - f1_m: 1.2473 - precision_m: 1.1002 - recall_m: 1.4461
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4341 - accuracy: 0.8334 - f1_m: 1.2422 - precision_m: 1.0945 - recall_m: 1.4419
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4295 - accuracy: 0.8354 - f1_m: 1.2391 - precision_m: 1.0904 - recall_m: 1.4405
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4276 - accuracy: 0.8365 - f1_m: 1.2362 - precision_m: 1.0871 - recall_m: 1.4385
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4244 - accuracy: 0.8366 - f1_m: 1.2338 - precision_m: 1.0852 - recall_m: 1.4360
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4222 - accuracy: 0.8384 - f1_m: 1.2312 - precision_m: 1.0828 - recall_m: 1.4329
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4197 - accuracy: 0.8395 - f1_m: 1.2286 - precision_m: 1.0786 - recall_m: 1.4328
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4173 - accuracy: 0.8405 - f1_m: 1.2281 - precision_m: 1.0787 - recall_m: 1.4309
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4165 - accuracy: 0.8404 - f1_m: 1.2238 - precision_m: 1.0744 - recall_m: 1.4276
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4139 - accuracy: 0.8412 - f1_m: 1.2221 - precision_m: 1.0727 - recall_m: 1.4260
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4119 - accuracy: 0.8408 - f1_m: 1.2204 - precision_m: 1.0703 - recall_m: 1.4253
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4105 - accuracy: 0.8422 - f1_m: 1.2193 - precision_m: 1.0692 - recall_m: 1.4244
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4093 - accuracy: 0.8420 - f1_m: 1.2219 - precision_m: 1.0720 - recall_m: 1.4269
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4083 - accuracy: 0.8432 - f1_m: 1.2184 - precision_m: 1.0679 - recall_m: 1.4242
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4054 - accuracy: 0.8433 - f1_m: 1.2182 - precision_m: 1.0679 - recall_m: 1.4234
Epoch 31/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4040 - accuracy: 0.8441 - f1_m: 1.2164 - precision_m: 1.0654 - recall_m: 1.4229
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4027 - accuracy: 0.8446 - f1_m: 1.2156 - precision_m: 1.0647 - recall_m: 1.4222
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4016 - accuracy: 0.8456 - f1_m: 1.2154 - precision_m: 1.0647 - recall_m: 1.4218
Epoch 34/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4012 - accuracy: 0.8446 - f1_m: 1.2137 - precision_m: 1.0626 - recall_m: 1.4210
Epoch 35/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4010 - accuracy: 0.8457 - f1_m: 1.2129 - precision_m: 1.0621 - recall_m: 1.4195
Epoch 36/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3985 - accuracy: 0.8462 - f1_m: 1.2122 - precision_m: 1.0613 - recall_m: 1.4190
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3970 - accuracy: 0.8460 - f1_m: 1.2119 - precision_m: 1.0610 - recall_m: 1.4190
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3959 - accuracy: 0.8469 - f1_m: 1.2104 - precision_m: 1.0591 - recall_m: 1.4176
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3942 - accuracy: 0.8470 - f1_m: 1.2108 - precision_m: 1.0594 - recall_m: 1.4185
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3948 - accuracy: 0.8470 - f1_m: 1.2077 - precision_m: 1.0563 - recall_m: 1.4158
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3930 - accuracy: 0.8478 - f1_m: 1.2094 - precision_m: 1.0585 - recall_m: 1.4164
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3907 - accuracy: 0.8487 - f1_m: 1.2074 - precision_m: 1.0560 - recall_m: 1.4153
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3908 - accuracy: 0.8479 - f1_m: 1.2076 - precision_m: 1.0558 - recall_m: 1.4162
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3890 - accuracy: 0.8489 - f1_m: 1.2056 - precision_m: 1.0539 - recall_m: 1.4141
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3886 - accuracy: 0.8489 - f1_m: 1.2050 - precision_m: 1.0534 - recall_m: 1.4134
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3886 - accuracy: 0.8486 - f1_m: 1.2044 - precision_m: 1.0530 - recall_m: 1.4126
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3879 - accuracy: 0.8490 - f1_m: 1.2070 - precision_m: 1.0554 - recall_m: 1.4156
Epoch 48/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3871 - accuracy: 0.8502 - f1_m: 1.2048 - precision_m: 1.0526 - recall_m: 1.4143
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3859 - accuracy: 0.8501 - f1_m: 1.2047 - precision_m: 1.0532 - recall_m: 1.4128
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3836 - accuracy: 0.8503 - f1_m: 1.2034 - precision_m: 1.0514 - recall_m: 1.4126
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3832 - accuracy: 0.8506 - f1_m: 1.2027 - precision_m: 1.0500 - recall_m: 1.4131
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3831 - accuracy: 0.8503 - f1_m: 1.2018 - precision_m: 1.0503 - recall_m: 1.4103
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3837 - accuracy: 0.8508 - f1_m: 1.2005 - precision_m: 1.0482 - recall_m: 1.4105
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3812 - accuracy: 0.8512 - f1_m: 1.2019 - precision_m: 1.0494 - recall_m: 1.4120
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3810 - accuracy: 0.8513 - f1_m: 1.2001 - precision_m: 1.0469 - recall_m: 1.4116
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3794 - accuracy: 0.8520 - f1_m: 1.2032 - precision_m: 1.0510 - recall_m: 1.4125
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3799 - accuracy: 0.8527 - f1_m: 1.1985 - precision_m: 1.0459 - recall_m: 1.4091
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3783 - accuracy: 0.8527 - f1_m: 1.2000 - precision_m: 1.0473 - recall_m: 1.4104
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3771 - accuracy: 0.8522 - f1_m: 1.1986 - precision_m: 1.0460 - recall_m: 1.4092
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3767 - accuracy: 0.8518 - f1_m: 1.2001 - precision_m: 1.0476 - recall_m: 1.4101
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3765 - accuracy: 0.8536 - f1_m: 1.1983 - precision_m: 1.0452 - recall_m: 1.4098
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3755 - accuracy: 0.8526 - f1_m: 1.1984 - precision_m: 1.0458 - recall_m: 1.4092
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3745 - accuracy: 0.8537 - f1_m: 1.1961 - precision_m: 1.0433 - recall_m: 1.4073
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3740 - accuracy: 0.8533 - f1_m: 1.1990 - precision_m: 1.0465 - recall_m: 1.4091
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3732 - accuracy: 0.8539 - f1_m: 1.1969 - precision_m: 1.0439 - recall_m: 1.4077
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3748 - accuracy: 0.8540 - f1_m: 1.1940 - precision_m: 1.0410 - recall_m: 1.4053
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3717 - accuracy: 0.8543 - f1_m: 1.1946 - precision_m: 1.0405 - recall_m: 1.4077
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3716 - accuracy: 0.8543 - f1_m: 1.1947 - precision_m: 1.0419 - recall_m: 1.4056
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3699 - accuracy: 0.8550 - f1_m: 1.1934 - precision_m: 1.0394 - recall_m: 1.4063
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3696 - accuracy: 0.8556 - f1_m: 1.1917 - precision_m: 1.0380 - recall_m: 1.4044
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3700 - accuracy: 0.8554 - f1_m: 1.1947 - precision_m: 1.0406 - recall_m: 1.4079
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3691 - accuracy: 0.8552 - f1_m: 1.1938 - precision_m: 1.0405 - recall_m: 1.4056
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3676 - accuracy: 0.8556 - f1_m: 1.1922 - precision_m: 1.0390 - recall_m: 1.4042
Epoch 74/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3682 - accuracy: 0.8551 - f1_m: 1.1931 - precision_m: 1.0396 - recall_m: 1.4054
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3670 - accuracy: 0.8564 - f1_m: 1.1930 - precision_m: 1.0393 - recall_m: 1.4055
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3668 - accuracy: 0.8563 - f1_m: 1.1909 - precision_m: 1.0372 - recall_m: 1.4039
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3665 - accuracy: 0.8569 - f1_m: 1.1921 - precision_m: 1.0384 - recall_m: 1.4054
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3670 - accuracy: 0.8561 - f1_m: 1.1927 - precision_m: 1.0390 - recall_m: 1.4053
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3650 - accuracy: 0.8571 - f1_m: 1.1902 - precision_m: 1.0361 - recall_m: 1.4036
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3641 - accuracy: 0.8574 - f1_m: 1.1900 - precision_m: 1.0362 - recall_m: 1.4031
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3641 - accuracy: 0.8577 - f1_m: 1.1904 - precision_m: 1.0365 - recall_m: 1.4037
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3634 - accuracy: 0.8573 - f1_m: 1.1881 - precision_m: 1.0337 - recall_m: 1.4022
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3625 - accuracy: 0.8580 - f1_m: 1.1888 - precision_m: 1.0347 - recall_m: 1.4027
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3630 - accuracy: 0.8572 - f1_m: 1.1878 - precision_m: 1.0337 - recall_m: 1.4018
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3612 - accuracy: 0.8579 - f1_m: 1.1886 - precision_m: 1.0345 - recall_m: 1.4024
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3617 - accuracy: 0.8577 - f1_m: 1.1892 - precision_m: 1.0358 - recall_m: 1.4017
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3623 - accuracy: 0.8575 - f1_m: 1.1884 - precision_m: 1.0339 - recall_m: 1.4026
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3603 - accuracy: 0.8584 - f1_m: 1.1848 - precision_m: 1.0305 - recall_m: 1.3988
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3615 - accuracy: 0.8577 - f1_m: 1.1871 - precision_m: 1.0331 - recall_m: 1.4006
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3591 - accuracy: 0.8594 - f1_m: 1.1839 - precision_m: 1.0298 - recall_m: 1.3979
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3595 - accuracy: 0.8589 - f1_m: 1.1868 - precision_m: 1.0323 - recall_m: 1.4009
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3600 - accuracy: 0.8586 - f1_m: 1.1880 - precision_m: 1.0334 - recall_m: 1.4027
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3599 - accuracy: 0.8592 - f1_m: 1.1860 - precision_m: 1.0324 - recall_m: 1.3991
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3570 - accuracy: 0.8597 - f1_m: 1.1842 - precision_m: 1.0298 - recall_m: 1.3986
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3572 - accuracy: 0.8595 - f1_m: 1.1838 - precision_m: 1.0294 - recall_m: 1.3984
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3582 - accuracy: 0.8598 - f1_m: 1.1837 - precision_m: 1.0292 - recall_m: 1.3984
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3563 - accuracy: 0.8598 - f1_m: 1.1839 - precision_m: 1.0295 - recall_m: 1.3983
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3575 - accuracy: 0.8595 - f1_m: 1.1833 - precision_m: 1.0289 - recall_m: 1.3976
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3558 - accuracy: 0.8595 - f1_m: 1.1838 - precision_m: 1.0292 - recall_m: 1.3986
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3556 - accuracy: 0.8598 - f1_m: 1.1825 - precision_m: 1.0281 - recall_m: 1.3971
2658/2658 [==============================] - 3s 950us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4266 - accuracy: 0.8465 - f1_m: 1.1754 - precision_m: 1.0229 - recall_m: 1.4019
score on test: 0.8464505672454834
5395/5395 [==============================] - 6s 1ms/step - loss: 0.3549 - accuracy: 0.8607 - f1_m: 1.1712 - precision_m: 1.0203 - recall_m: 1.3958
score on train: 0.86066734790802

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.7407588822697762

Noise range 2

noise4 = 4
X_train_mod4 = modify_dataset(X_train, variables_to_poison, noise4)

X_train_mod4

array([[ 1.79214449, -0.2082886 , -3.08683061, ..., -0.8410797 ,
         0.31502158, -0.12070382],
       [-0.57638416,  1.78686151,  9.08331212, ...,  1.29036785,
        -1.94185765, -0.12070382],
       [-1.0569863 , -0.20828827, -7.25638579, ...,  0.34067846,
        -1.6484505 , -0.12070382],
       ...,
       [-0.47187522, -0.06694436,  4.49053066, ..., -0.72809357,
        -2.84974832, -0.12070382],
       [-0.1666061 , -0.18783648, -3.83237825, ..., -2.83790831,
         0.82965585, -0.12070382],
       [-0.6757627 , -0.20011705,  2.61409594, ...,  2.11692769,
         2.65200419, -0.12070382]])

from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod4, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod4, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['4']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.8924471938655502
score on train: 0.8928575566239935
              precision    recall  f1-score   support

           0       0.93      0.76      0.84     30772
           1       0.88      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.90      0.86      0.88     85028
weighted avg       0.90      0.89      0.89     85028

# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9538

rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod4, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod4, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['4']=acc

<ipython-input-62-e1f13e046cab>:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod4, y_train)

score on test: 0.866020604977184
score on train: 0.9475931182297399
              precision    recall  f1-score   support

           0       0.75      0.94      0.83     30772
           1       0.96      0.83      0.89     54256

    accuracy                           0.87     85028
   macro avg       0.86      0.88      0.86     85028
weighted avg       0.88      0.87      0.87     85028

rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9654

xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod4, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod4, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['4']=acc

score on test: 0.9683751234887331
score on train: 0.9921682210508023
              precision    recall  f1-score   support

           0       0.96      0.95      0.96     30772
           1       0.97      0.98      0.98     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.96      0.97     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9959

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod4, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod4,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores=model.evaluate(X_test, y_test, verbose=0)
res_nn_b['4']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.2533 - accuracy: 0.8847 - f1_m: 0.9129 - precision_m: 0.8802 - recall_m: 0.9499
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1917 - accuracy: 0.9118 - f1_m: 0.9320 - precision_m: 0.9140 - recall_m: 0.9518
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1769 - accuracy: 0.9193 - f1_m: 0.9375 - precision_m: 0.9235 - recall_m: 0.9528
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1577 - accuracy: 0.9292 - f1_m: 0.9449 - precision_m: 0.9358 - recall_m: 0.9550
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1299 - accuracy: 0.9438 - f1_m: 0.9560 - precision_m: 0.9520 - recall_m: 0.9607
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1103 - accuracy: 0.9526 - f1_m: 0.9628 - precision_m: 0.9613 - recall_m: 0.9649
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1034 - accuracy: 0.9558 - f1_m: 0.9653 - precision_m: 0.9652 - recall_m: 0.9658
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0978 - accuracy: 0.9583 - f1_m: 0.9672 - precision_m: 0.9676 - recall_m: 0.9674
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0947 - accuracy: 0.9596 - f1_m: 0.9682 - precision_m: 0.9691 - recall_m: 0.9678
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0914 - accuracy: 0.9610 - f1_m: 0.9693 - precision_m: 0.9700 - recall_m: 0.9689
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0895 - accuracy: 0.9617 - f1_m: 0.9699 - precision_m: 0.9714 - recall_m: 0.9688
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0878 - accuracy: 0.9625 - f1_m: 0.9704 - precision_m: 0.9722 - recall_m: 0.9691
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0855 - accuracy: 0.9636 - f1_m: 0.9713 - precision_m: 0.9727 - recall_m: 0.9703
Epoch 14/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0837 - accuracy: 0.9646 - f1_m: 0.9721 - precision_m: 0.9738 - recall_m: 0.9708
Epoch 15/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0809 - accuracy: 0.9662 - f1_m: 0.9733 - precision_m: 0.9753 - recall_m: 0.9717
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0803 - accuracy: 0.9660 - f1_m: 0.9732 - precision_m: 0.9752 - recall_m: 0.9716
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0786 - accuracy: 0.9665 - f1_m: 0.9736 - precision_m: 0.9759 - recall_m: 0.9718
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0769 - accuracy: 0.9674 - f1_m: 0.9743 - precision_m: 0.9766 - recall_m: 0.9723
Epoch 19/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0758 - accuracy: 0.9677 - f1_m: 0.9745 - precision_m: 0.9768 - recall_m: 0.9726
Epoch 20/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0745 - accuracy: 0.9687 - f1_m: 0.9754 - precision_m: 0.9779 - recall_m: 0.9732
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0731 - accuracy: 0.9689 - f1_m: 0.9754 - precision_m: 0.9781 - recall_m: 0.9731
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0729 - accuracy: 0.9694 - f1_m: 0.9759 - precision_m: 0.9786 - recall_m: 0.9735
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0723 - accuracy: 0.9695 - f1_m: 0.9759 - precision_m: 0.9787 - recall_m: 0.9735
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0708 - accuracy: 0.9701 - f1_m: 0.9765 - precision_m: 0.9790 - recall_m: 0.9743
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0699 - accuracy: 0.9707 - f1_m: 0.9769 - precision_m: 0.9798 - recall_m: 0.9743
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0693 - accuracy: 0.9709 - f1_m: 0.9771 - precision_m: 0.9801 - recall_m: 0.9744
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0694 - accuracy: 0.9709 - f1_m: 0.9770 - precision_m: 0.9800 - recall_m: 0.9744
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0682 - accuracy: 0.9710 - f1_m: 0.9772 - precision_m: 0.9801 - recall_m: 0.9745
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0669 - accuracy: 0.9717 - f1_m: 0.9777 - precision_m: 0.9810 - recall_m: 0.9748
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0664 - accuracy: 0.9721 - f1_m: 0.9780 - precision_m: 0.9814 - recall_m: 0.9749
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0661 - accuracy: 0.9720 - f1_m: 0.9778 - precision_m: 0.9810 - recall_m: 0.9750
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0654 - accuracy: 0.9723 - f1_m: 0.9782 - precision_m: 0.9813 - recall_m: 0.9753
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0640 - accuracy: 0.9730 - f1_m: 0.9787 - precision_m: 0.9819 - recall_m: 0.9759
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0641 - accuracy: 0.9727 - f1_m: 0.9784 - precision_m: 0.9818 - recall_m: 0.9753
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0640 - accuracy: 0.9727 - f1_m: 0.9785 - precision_m: 0.9817 - recall_m: 0.9756
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0624 - accuracy: 0.9736 - f1_m: 0.9791 - precision_m: 0.9824 - recall_m: 0.9761
Epoch 37/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0630 - accuracy: 0.9735 - f1_m: 0.9791 - precision_m: 0.9821 - recall_m: 0.9764
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0618 - accuracy: 0.9738 - f1_m: 0.9793 - precision_m: 0.9827 - recall_m: 0.9762
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0617 - accuracy: 0.9738 - f1_m: 0.9793 - precision_m: 0.9826 - recall_m: 0.9764
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0604 - accuracy: 0.9742 - f1_m: 0.9797 - precision_m: 0.9827 - recall_m: 0.9769
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0607 - accuracy: 0.9743 - f1_m: 0.9797 - precision_m: 0.9830 - recall_m: 0.9768
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0610 - accuracy: 0.9747 - f1_m: 0.9800 - precision_m: 0.9835 - recall_m: 0.9769
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0595 - accuracy: 0.9747 - f1_m: 0.9801 - precision_m: 0.9835 - recall_m: 0.9769
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0589 - accuracy: 0.9754 - f1_m: 0.9806 - precision_m: 0.9840 - recall_m: 0.9774
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0584 - accuracy: 0.9754 - f1_m: 0.9806 - precision_m: 0.9839 - recall_m: 0.9775
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0583 - accuracy: 0.9755 - f1_m: 0.9807 - precision_m: 0.9840 - recall_m: 0.9777
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0578 - accuracy: 0.9755 - f1_m: 0.9807 - precision_m: 0.9839 - recall_m: 0.9777
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0575 - accuracy: 0.9762 - f1_m: 0.9812 - precision_m: 0.9843 - recall_m: 0.9784
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0565 - accuracy: 0.9763 - f1_m: 0.9813 - precision_m: 0.9846 - recall_m: 0.9783
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0564 - accuracy: 0.9762 - f1_m: 0.9812 - precision_m: 0.9843 - recall_m: 0.9783
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0568 - accuracy: 0.9761 - f1_m: 0.9812 - precision_m: 0.9842 - recall_m: 0.9784
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0552 - accuracy: 0.9766 - f1_m: 0.9815 - precision_m: 0.9848 - recall_m: 0.9784
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0558 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9848 - recall_m: 0.9782
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0547 - accuracy: 0.9770 - f1_m: 0.9819 - precision_m: 0.9852 - recall_m: 0.9788
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0548 - accuracy: 0.9769 - f1_m: 0.9818 - precision_m: 0.9849 - recall_m: 0.9790
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0543 - accuracy: 0.9772 - f1_m: 0.9820 - precision_m: 0.9851 - recall_m: 0.9792
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0547 - accuracy: 0.9771 - f1_m: 0.9819 - precision_m: 0.9849 - recall_m: 0.9792
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0540 - accuracy: 0.9775 - f1_m: 0.9823 - precision_m: 0.9855 - recall_m: 0.9793
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0534 - accuracy: 0.9776 - f1_m: 0.9823 - precision_m: 0.9853 - recall_m: 0.9795
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0525 - accuracy: 0.9777 - f1_m: 0.9824 - precision_m: 0.9856 - recall_m: 0.9795
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0524 - accuracy: 0.9777 - f1_m: 0.9825 - precision_m: 0.9856 - recall_m: 0.9795
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0518 - accuracy: 0.9782 - f1_m: 0.9828 - precision_m: 0.9857 - recall_m: 0.9801
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0525 - accuracy: 0.9782 - f1_m: 0.9828 - precision_m: 0.9860 - recall_m: 0.9799
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0515 - accuracy: 0.9783 - f1_m: 0.9828 - precision_m: 0.9861 - recall_m: 0.9799
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0516 - accuracy: 0.9785 - f1_m: 0.9830 - precision_m: 0.9861 - recall_m: 0.9802
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0510 - accuracy: 0.9783 - f1_m: 0.9829 - precision_m: 0.9858 - recall_m: 0.9803
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0505 - accuracy: 0.9787 - f1_m: 0.9832 - precision_m: 0.9866 - recall_m: 0.9801
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0498 - accuracy: 0.9792 - f1_m: 0.9836 - precision_m: 0.9867 - recall_m: 0.9808
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0496 - accuracy: 0.9789 - f1_m: 0.9833 - precision_m: 0.9863 - recall_m: 0.9805
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0489 - accuracy: 0.9795 - f1_m: 0.9838 - precision_m: 0.9866 - recall_m: 0.9813
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0491 - accuracy: 0.9794 - f1_m: 0.9837 - precision_m: 0.9869 - recall_m: 0.9809
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0500 - accuracy: 0.9792 - f1_m: 0.9835 - precision_m: 0.9866 - recall_m: 0.9807
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0482 - accuracy: 0.9793 - f1_m: 0.9837 - precision_m: 0.9866 - recall_m: 0.9810
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0492 - accuracy: 0.9795 - f1_m: 0.9839 - precision_m: 0.9869 - recall_m: 0.9811
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0480 - accuracy: 0.9798 - f1_m: 0.9841 - precision_m: 0.9873 - recall_m: 0.9812
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9799 - f1_m: 0.9842 - precision_m: 0.9871 - recall_m: 0.9815
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0486 - accuracy: 0.9794 - f1_m: 0.9837 - precision_m: 0.9867 - recall_m: 0.9809
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0471 - accuracy: 0.9796 - f1_m: 0.9839 - precision_m: 0.9868 - recall_m: 0.9812
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0474 - accuracy: 0.9800 - f1_m: 0.9843 - precision_m: 0.9870 - recall_m: 0.9817
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0479 - accuracy: 0.9800 - f1_m: 0.9843 - precision_m: 0.9871 - recall_m: 0.9817
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0468 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9876 - recall_m: 0.9823
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9806 - f1_m: 0.9847 - precision_m: 0.9874 - recall_m: 0.9821
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0454 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9878 - recall_m: 0.9822
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0475 - accuracy: 0.9803 - f1_m: 0.9844 - precision_m: 0.9874 - recall_m: 0.9816
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0457 - accuracy: 0.9810 - f1_m: 0.9850 - precision_m: 0.9879 - recall_m: 0.9824
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0452 - accuracy: 0.9812 - f1_m: 0.9852 - precision_m: 0.9882 - recall_m: 0.9824
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0454 - accuracy: 0.9810 - f1_m: 0.9850 - precision_m: 0.9878 - recall_m: 0.9824
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0451 - accuracy: 0.9812 - f1_m: 0.9852 - precision_m: 0.9879 - recall_m: 0.9827
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0445 - accuracy: 0.9813 - f1_m: 0.9853 - precision_m: 0.9882 - recall_m: 0.9825
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0454 - accuracy: 0.9812 - f1_m: 0.9851 - precision_m: 0.9879 - recall_m: 0.9825
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0435 - accuracy: 0.9818 - f1_m: 0.9857 - precision_m: 0.9888 - recall_m: 0.9828
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0448 - accuracy: 0.9812 - f1_m: 0.9851 - precision_m: 0.9879 - recall_m: 0.9826
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0442 - accuracy: 0.9817 - f1_m: 0.9855 - precision_m: 0.9885 - recall_m: 0.9828
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0433 - accuracy: 0.9818 - f1_m: 0.9857 - precision_m: 0.9884 - recall_m: 0.9831
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0432 - accuracy: 0.9820 - f1_m: 0.9858 - precision_m: 0.9885 - recall_m: 0.9833
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0434 - accuracy: 0.9817 - f1_m: 0.9856 - precision_m: 0.9883 - recall_m: 0.9831
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0426 - accuracy: 0.9822 - f1_m: 0.9860 - precision_m: 0.9888 - recall_m: 0.9835
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0431 - accuracy: 0.9818 - f1_m: 0.9857 - precision_m: 0.9882 - recall_m: 0.9834
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0420 - accuracy: 0.9821 - f1_m: 0.9858 - precision_m: 0.9884 - recall_m: 0.9835
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0429 - accuracy: 0.9821 - f1_m: 0.9858 - precision_m: 0.9886 - recall_m: 0.9834
2658/2658 [==============================] - 2s 887us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0824 - accuracy: 0.9710 - f1_m: 0.9767 - precision_m: 0.9812 - recall_m: 0.9734
score on test: 0.971021294593811
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0362 - accuracy: 0.9849 - f1_m: 0.9879 - precision_m: 0.9920 - recall_m: 0.9844
score on train: 0.9849099516868591

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod4, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod4, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['4']=acc

score on test: 0.805040692477772
score on train: 0.8268840873544575
              precision    recall  f1-score   support

           0       0.90      0.94      0.92     30772
           1       0.98      0.96      0.97     19423
           2       0.59      0.83      0.69     14709
           3       0.69      0.59      0.63      7921
           4       0.44      0.13      0.20      5357
           5       0.68      0.72      0.70      4578
           6       0.69      0.02      0.05       909
           7       0.16      0.02      0.03       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.81     85028
   macro avg       0.51      0.42      0.42     85028
weighted avg       0.79      0.81      0.78     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(dtree1, X_train_mod4, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod4, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod4, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['4']=acc

<ipython-input-70-7575b483ca04>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod4, y_multi_train)

score on test: 0.7454603189537564
score on train: 0.8181544343393384
              precision    recall  f1-score   support

           0       0.67      0.97      0.79     30772
           1       1.00      0.96      0.98     19423
           2       0.65      0.69      0.67     14709
           3       0.73      0.44      0.55      7921
           4       0.84      0.01      0.01      5357
           5       0.82      0.31      0.45      4578
           6       1.00      0.00      0.00       909
           7       1.00      0.00      0.01       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.75     85028
   macro avg       0.67      0.34      0.35     85028
weighted avg       0.77      0.75      0.70     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod4, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod4, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod4, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['4']=acc

score on test: 0.8540010349531919
score on train: 0.9088107513178474
              precision    recall  f1-score   support

           0       0.95      0.98      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.69      0.78      0.73     14709
           3       0.78      0.79      0.79      7921
           4       0.39      0.38      0.39      5357
           5       0.88      0.77      0.82      4578
           6       0.65      0.11      0.19       909
           7       0.69      0.08      0.14       780
           8       0.69      0.33      0.44       513
           9       0.64      0.11      0.18        66

    accuracy                           0.85     85028
   macro avg       0.74      0.53      0.56     85028
weighted avg       0.85      0.85      0.85     85028

plot_ROC_curve(xgb2, X_train_mod4, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod4,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
scores=model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['4']=scores[1]
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod4,y_multi_train)[1]))
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_7"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_28 (Dense)            (None, 128)               5504      
                                                                 
 dense_29 (Dense)            (None, 64)                8256      
                                                                 
 dense_30 (Dense)            (None, 64)                4160      
                                                                 
 dense_31 (Dense)            (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.8790 - accuracy: 0.6797 - f1_m: 1.7411 - precision_m: 1104522.0000 - recall_m: 1.7011
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6861 - accuracy: 0.7446 - f1_m: 1.5584 - precision_m: 1.4889 - recall_m: 1.6441
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6172 - accuracy: 0.7692 - f1_m: 1.4678 - precision_m: 1.3604 - recall_m: 1.6017
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5768 - accuracy: 0.7834 - f1_m: 1.4071 - precision_m: 1.2841 - recall_m: 1.5638
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5521 - accuracy: 0.7918 - f1_m: 1.3691 - precision_m: 1.2396 - recall_m: 1.5359
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5342 - accuracy: 0.7979 - f1_m: 1.3445 - precision_m: 1.2113 - recall_m: 1.5176
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5210 - accuracy: 0.8031 - f1_m: 1.3300 - precision_m: 1.1936 - recall_m: 1.5085
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5067 - accuracy: 0.8082 - f1_m: 1.3147 - precision_m: 1.1768 - recall_m: 1.4959
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4923 - accuracy: 0.8135 - f1_m: 1.2981 - precision_m: 1.1575 - recall_m: 1.4842
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4782 - accuracy: 0.8191 - f1_m: 1.2821 - precision_m: 1.1388 - recall_m: 1.4733
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4681 - accuracy: 0.8229 - f1_m: 1.2708 - precision_m: 1.1272 - recall_m: 1.4627
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4594 - accuracy: 0.8259 - f1_m: 1.2622 - precision_m: 1.1173 - recall_m: 1.4564
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4529 - accuracy: 0.8274 - f1_m: 1.2550 - precision_m: 1.1087 - recall_m: 1.4520
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4484 - accuracy: 0.8304 - f1_m: 1.2514 - precision_m: 1.1049 - recall_m: 1.4494
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4438 - accuracy: 0.8306 - f1_m: 1.2454 - precision_m: 1.0984 - recall_m: 1.4444
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4399 - accuracy: 0.8316 - f1_m: 1.2447 - precision_m: 1.0973 - recall_m: 1.4438
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4375 - accuracy: 0.8327 - f1_m: 1.2438 - precision_m: 1.0969 - recall_m: 1.4424
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4349 - accuracy: 0.8338 - f1_m: 1.2386 - precision_m: 1.0906 - recall_m: 1.4392
Epoch 19/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4301 - accuracy: 0.8349 - f1_m: 1.2396 - precision_m: 1.0914 - recall_m: 1.4403
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4288 - accuracy: 0.8357 - f1_m: 1.2363 - precision_m: 1.0875 - recall_m: 1.4379
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4266 - accuracy: 0.8361 - f1_m: 1.2368 - precision_m: 1.0884 - recall_m: 1.4385
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4244 - accuracy: 0.8367 - f1_m: 1.2338 - precision_m: 1.0848 - recall_m: 1.4361
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4220 - accuracy: 0.8373 - f1_m: 1.2324 - precision_m: 1.0835 - recall_m: 1.4351
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4207 - accuracy: 0.8386 - f1_m: 1.2296 - precision_m: 1.0808 - recall_m: 1.4324
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4179 - accuracy: 0.8392 - f1_m: 1.2290 - precision_m: 1.0799 - recall_m: 1.4316
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4177 - accuracy: 0.8388 - f1_m: 1.2289 - precision_m: 1.0791 - recall_m: 1.4328
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4157 - accuracy: 0.8399 - f1_m: 1.2280 - precision_m: 1.0780 - recall_m: 1.4323
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4130 - accuracy: 0.8413 - f1_m: 1.2274 - precision_m: 1.0780 - recall_m: 1.4306
Epoch 29/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4116 - accuracy: 0.8408 - f1_m: 1.2253 - precision_m: 1.0751 - recall_m: 1.4301
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4093 - accuracy: 0.8411 - f1_m: 1.2245 - precision_m: 1.0739 - recall_m: 1.4298
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4084 - accuracy: 0.8419 - f1_m: 1.2229 - precision_m: 1.0721 - recall_m: 1.4286
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4070 - accuracy: 0.8423 - f1_m: 1.2217 - precision_m: 1.0704 - recall_m: 1.4285
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4054 - accuracy: 0.8428 - f1_m: 1.2206 - precision_m: 1.0690 - recall_m: 1.4279
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4040 - accuracy: 0.8441 - f1_m: 1.2206 - precision_m: 1.0700 - recall_m: 1.4263
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4029 - accuracy: 0.8443 - f1_m: 1.2181 - precision_m: 1.0669 - recall_m: 1.4249
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4014 - accuracy: 0.8453 - f1_m: 1.2181 - precision_m: 1.0672 - recall_m: 1.4247
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3998 - accuracy: 0.8452 - f1_m: 1.2165 - precision_m: 1.0650 - recall_m: 1.4239
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3992 - accuracy: 0.8452 - f1_m: 1.2169 - precision_m: 1.0650 - recall_m: 1.4255
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3977 - accuracy: 0.8460 - f1_m: 1.2166 - precision_m: 1.0644 - recall_m: 1.4253
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3969 - accuracy: 0.8466 - f1_m: 1.2135 - precision_m: 1.0616 - recall_m: 1.4220
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3956 - accuracy: 0.8463 - f1_m: 1.2151 - precision_m: 1.0634 - recall_m: 1.4228
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3941 - accuracy: 0.8469 - f1_m: 1.2131 - precision_m: 1.0610 - recall_m: 1.4223
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3941 - accuracy: 0.8470 - f1_m: 1.2115 - precision_m: 1.0591 - recall_m: 1.4211
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3922 - accuracy: 0.8478 - f1_m: 1.2141 - precision_m: 1.0619 - recall_m: 1.4227
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3910 - accuracy: 0.8476 - f1_m: 1.2105 - precision_m: 1.0576 - recall_m: 1.4210
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3914 - accuracy: 0.8479 - f1_m: 1.2103 - precision_m: 1.0582 - recall_m: 1.4194
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3883 - accuracy: 0.8489 - f1_m: 1.2106 - precision_m: 1.0581 - recall_m: 1.4200
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3893 - accuracy: 0.8490 - f1_m: 1.2110 - precision_m: 1.0577 - recall_m: 1.4216
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3874 - accuracy: 0.8496 - f1_m: 1.2102 - precision_m: 1.0583 - recall_m: 1.4190
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3876 - accuracy: 0.8486 - f1_m: 1.2109 - precision_m: 1.0582 - recall_m: 1.4207
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3872 - accuracy: 0.8492 - f1_m: 1.2095 - precision_m: 1.0566 - recall_m: 1.4198
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3862 - accuracy: 0.8501 - f1_m: 1.2096 - precision_m: 1.0574 - recall_m: 1.4190
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3828 - accuracy: 0.8510 - f1_m: 1.2060 - precision_m: 1.0531 - recall_m: 1.4166
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3836 - accuracy: 0.8503 - f1_m: 1.2075 - precision_m: 1.0544 - recall_m: 1.4187
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3828 - accuracy: 0.8505 - f1_m: 1.2069 - precision_m: 1.0542 - recall_m: 1.4173
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3821 - accuracy: 0.8514 - f1_m: 1.2049 - precision_m: 1.0520 - recall_m: 1.4158
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3810 - accuracy: 0.8514 - f1_m: 1.2065 - precision_m: 1.0537 - recall_m: 1.4168
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3806 - accuracy: 0.8513 - f1_m: 1.2064 - precision_m: 1.0536 - recall_m: 1.4169
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3791 - accuracy: 0.8520 - f1_m: 1.2035 - precision_m: 1.0500 - recall_m: 1.4151
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3784 - accuracy: 0.8525 - f1_m: 1.2071 - precision_m: 1.0541 - recall_m: 1.4180
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3792 - accuracy: 0.8514 - f1_m: 1.2039 - precision_m: 1.0507 - recall_m: 1.4153
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3778 - accuracy: 0.8527 - f1_m: 1.2028 - precision_m: 1.0494 - recall_m: 1.4143
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3777 - accuracy: 0.8528 - f1_m: 1.2020 - precision_m: 1.0477 - recall_m: 1.4155
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3768 - accuracy: 0.8526 - f1_m: 1.2046 - precision_m: 1.0510 - recall_m: 1.4165
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3758 - accuracy: 0.8528 - f1_m: 1.2019 - precision_m: 1.0486 - recall_m: 1.4135
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3749 - accuracy: 0.8529 - f1_m: 1.2019 - precision_m: 1.0486 - recall_m: 1.4136
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3750 - accuracy: 0.8528 - f1_m: 1.2008 - precision_m: 1.0470 - recall_m: 1.4129
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3746 - accuracy: 0.8535 - f1_m: 1.2024 - precision_m: 1.0483 - recall_m: 1.4151
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3729 - accuracy: 0.8543 - f1_m: 1.1997 - precision_m: 1.0461 - recall_m: 1.4121
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3733 - accuracy: 0.8536 - f1_m: 1.2013 - precision_m: 1.0478 - recall_m: 1.4133
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3722 - accuracy: 0.8541 - f1_m: 1.1997 - precision_m: 1.0454 - recall_m: 1.4127
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3718 - accuracy: 0.8542 - f1_m: 1.1980 - precision_m: 1.0442 - recall_m: 1.4107
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3730 - accuracy: 0.8536 - f1_m: 1.2004 - precision_m: 1.0460 - recall_m: 1.4140
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3703 - accuracy: 0.8542 - f1_m: 1.2002 - precision_m: 1.0465 - recall_m: 1.4129
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3702 - accuracy: 0.8548 - f1_m: 1.2013 - precision_m: 1.0483 - recall_m: 1.4123
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3690 - accuracy: 0.8555 - f1_m: 1.1980 - precision_m: 1.0440 - recall_m: 1.4107
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3693 - accuracy: 0.8555 - f1_m: 1.1971 - precision_m: 1.0435 - recall_m: 1.4094
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3680 - accuracy: 0.8558 - f1_m: 1.1981 - precision_m: 1.0432 - recall_m: 1.4125
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3684 - accuracy: 0.8548 - f1_m: 1.1977 - precision_m: 1.0438 - recall_m: 1.4107
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3680 - accuracy: 0.8554 - f1_m: 1.1987 - precision_m: 1.0450 - recall_m: 1.4111
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3671 - accuracy: 0.8557 - f1_m: 1.1990 - precision_m: 1.0453 - recall_m: 1.4115
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3662 - accuracy: 0.8563 - f1_m: 1.1978 - precision_m: 1.0444 - recall_m: 1.4094
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3658 - accuracy: 0.8568 - f1_m: 1.1948 - precision_m: 1.0408 - recall_m: 1.4078
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3649 - accuracy: 0.8568 - f1_m: 1.1967 - precision_m: 1.0424 - recall_m: 1.4099
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3651 - accuracy: 0.8565 - f1_m: 1.1961 - precision_m: 1.0417 - recall_m: 1.4101
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3651 - accuracy: 0.8572 - f1_m: 1.1939 - precision_m: 1.0396 - recall_m: 1.4074
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3636 - accuracy: 0.8569 - f1_m: 1.1938 - precision_m: 1.0394 - recall_m: 1.4074
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3631 - accuracy: 0.8573 - f1_m: 1.1951 - precision_m: 1.0411 - recall_m: 1.4085
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3636 - accuracy: 0.8573 - f1_m: 1.1944 - precision_m: 1.0400 - recall_m: 1.4078
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3629 - accuracy: 0.8574 - f1_m: 1.1927 - precision_m: 1.0384 - recall_m: 1.4066
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3633 - accuracy: 0.8572 - f1_m: 1.1939 - precision_m: 1.0394 - recall_m: 1.4079
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3609 - accuracy: 0.8584 - f1_m: 1.1927 - precision_m: 1.0382 - recall_m: 1.4069
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3624 - accuracy: 0.8576 - f1_m: 1.1925 - precision_m: 1.0383 - recall_m: 1.4061
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3610 - accuracy: 0.8579 - f1_m: 1.1928 - precision_m: 1.0384 - recall_m: 1.4070
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3599 - accuracy: 0.8583 - f1_m: 1.1927 - precision_m: 1.0385 - recall_m: 1.4065
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3598 - accuracy: 0.8585 - f1_m: 1.1917 - precision_m: 1.0373 - recall_m: 1.4056
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3597 - accuracy: 0.8578 - f1_m: 1.1946 - precision_m: 1.0398 - recall_m: 1.4090
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3597 - accuracy: 0.8582 - f1_m: 1.1906 - precision_m: 1.0357 - recall_m: 1.4055
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3586 - accuracy: 0.8591 - f1_m: 1.1914 - precision_m: 1.0366 - recall_m: 1.4061
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3587 - accuracy: 0.8589 - f1_m: 1.1915 - precision_m: 1.0364 - recall_m: 1.4066
2658/2658 [==============================] - 2s 903us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4199 - accuracy: 0.8460 - f1_m: 1.1838 - precision_m: 1.0410 - recall_m: 1.3941
score on test: 0.8460271954536438
5395/5395 [==============================] - 7s 1ms/step - loss: 0.3551 - accuracy: 0.8600 - f1_m: 1.1890 - precision_m: 1.0446 - recall_m: 1.4020
score on train: 0.8600416779518127

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.7593989598872701

Noise range 2.5

noise5 = 5
X_train_mod5 = modify_dataset(X_train, variables_to_poison, noise5)

X_train_mod5

array([[ 1.79214449, -0.2082886 ,  4.79472788, ..., -1.0619848 ,
         4.05578695, -0.12070382],
       [-0.57638416,  1.78686151, -0.21788858, ..., -2.81272213,
        -5.94086682, -0.12070382],
       [-1.0569863 , -0.20828827, -9.28511035, ...,  2.29888573,
        -3.32639488, -0.12070382],
       ...,
       [-0.47187522, -0.06694436,  1.08694708, ..., -5.93883527,
        -4.81004228, -0.12070382],
       [-0.1666061 , -0.18783648, -2.869486  , ...,  9.93929494,
         4.69624051, -0.12070382],
       [-0.6757627 , -0.20011705,  4.21417205, ...,  2.68682551,
        -2.92994458, -0.12070382]])

from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod5, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod5, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['5']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.8921178905772216
score on train: 0.8928633493599026
              precision    recall  f1-score   support

           0       0.93      0.76      0.84     30772
           1       0.88      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.90      0.86      0.88     85028
weighted avg       0.90      0.89      0.89     85028

# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9534

rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod5, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod5, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['5']=acc

<ipython-input-79-9b10e58482f5>:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod5, y_train)

score on test: 0.8765347885402456
score on train: 0.9476858020042866
              precision    recall  f1-score   support

           0       0.78      0.93      0.84     30772
           1       0.95      0.85      0.90     54256

    accuracy                           0.88     85028
   macro avg       0.86      0.89      0.87     85028
weighted avg       0.89      0.88      0.88     85028

rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9708

xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod5, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod5, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['5']=acc

score on test: 0.9703509432187044
score on train: 0.991884376991253
              precision    recall  f1-score   support

           0       0.97      0.95      0.96     30772
           1       0.97      0.98      0.98     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.97      0.97     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9962

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod5, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod5,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores=model.evaluate(X_test, y_test, verbose=0)
res_nn_b['5']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.2658 - accuracy: 0.8803 - f1_m: 0.9100 - precision_m: 0.8756 - recall_m: 0.9487
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.2024 - accuracy: 0.9059 - f1_m: 0.9280 - precision_m: 0.9051 - recall_m: 0.9531
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1791 - accuracy: 0.9160 - f1_m: 0.9350 - precision_m: 0.9217 - recall_m: 0.9497
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1621 - accuracy: 0.9240 - f1_m: 0.9408 - precision_m: 0.9324 - recall_m: 0.9503
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1491 - accuracy: 0.9313 - f1_m: 0.9462 - precision_m: 0.9411 - recall_m: 0.9522
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1392 - accuracy: 0.9380 - f1_m: 0.9514 - precision_m: 0.9484 - recall_m: 0.9550
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1260 - accuracy: 0.9449 - f1_m: 0.9567 - precision_m: 0.9557 - recall_m: 0.9584
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1117 - accuracy: 0.9522 - f1_m: 0.9624 - precision_m: 0.9622 - recall_m: 0.9633
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1011 - accuracy: 0.9568 - f1_m: 0.9661 - precision_m: 0.9659 - recall_m: 0.9667
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0957 - accuracy: 0.9593 - f1_m: 0.9679 - precision_m: 0.9681 - recall_m: 0.9683
Epoch 11/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0914 - accuracy: 0.9612 - f1_m: 0.9695 - precision_m: 0.9703 - recall_m: 0.9690
Epoch 12/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0890 - accuracy: 0.9624 - f1_m: 0.9704 - precision_m: 0.9716 - recall_m: 0.9697
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0868 - accuracy: 0.9633 - f1_m: 0.9711 - precision_m: 0.9722 - recall_m: 0.9706
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0843 - accuracy: 0.9644 - f1_m: 0.9719 - precision_m: 0.9737 - recall_m: 0.9705
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0823 - accuracy: 0.9653 - f1_m: 0.9726 - precision_m: 0.9743 - recall_m: 0.9713
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0812 - accuracy: 0.9658 - f1_m: 0.9731 - precision_m: 0.9752 - recall_m: 0.9714
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0794 - accuracy: 0.9664 - f1_m: 0.9735 - precision_m: 0.9757 - recall_m: 0.9718
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0777 - accuracy: 0.9668 - f1_m: 0.9738 - precision_m: 0.9762 - recall_m: 0.9719
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0763 - accuracy: 0.9675 - f1_m: 0.9744 - precision_m: 0.9770 - recall_m: 0.9721
Epoch 20/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0759 - accuracy: 0.9677 - f1_m: 0.9745 - precision_m: 0.9772 - recall_m: 0.9722
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0741 - accuracy: 0.9681 - f1_m: 0.9749 - precision_m: 0.9773 - recall_m: 0.9728
Epoch 22/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0736 - accuracy: 0.9685 - f1_m: 0.9752 - precision_m: 0.9778 - recall_m: 0.9729
Epoch 23/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0730 - accuracy: 0.9692 - f1_m: 0.9757 - precision_m: 0.9786 - recall_m: 0.9732
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0720 - accuracy: 0.9695 - f1_m: 0.9759 - precision_m: 0.9786 - recall_m: 0.9737
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0709 - accuracy: 0.9700 - f1_m: 0.9763 - precision_m: 0.9792 - recall_m: 0.9737
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0702 - accuracy: 0.9703 - f1_m: 0.9766 - precision_m: 0.9796 - recall_m: 0.9738
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0698 - accuracy: 0.9704 - f1_m: 0.9766 - precision_m: 0.9796 - recall_m: 0.9740
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0687 - accuracy: 0.9713 - f1_m: 0.9773 - precision_m: 0.9800 - recall_m: 0.9750
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0676 - accuracy: 0.9715 - f1_m: 0.9775 - precision_m: 0.9805 - recall_m: 0.9749
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0675 - accuracy: 0.9713 - f1_m: 0.9773 - precision_m: 0.9803 - recall_m: 0.9746
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0661 - accuracy: 0.9722 - f1_m: 0.9781 - precision_m: 0.9811 - recall_m: 0.9753
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0661 - accuracy: 0.9717 - f1_m: 0.9776 - precision_m: 0.9805 - recall_m: 0.9751
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0652 - accuracy: 0.9723 - f1_m: 0.9781 - precision_m: 0.9813 - recall_m: 0.9753
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0650 - accuracy: 0.9724 - f1_m: 0.9782 - precision_m: 0.9814 - recall_m: 0.9753
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0635 - accuracy: 0.9732 - f1_m: 0.9789 - precision_m: 0.9820 - recall_m: 0.9761
Epoch 36/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0627 - accuracy: 0.9735 - f1_m: 0.9791 - precision_m: 0.9825 - recall_m: 0.9760
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0632 - accuracy: 0.9731 - f1_m: 0.9788 - precision_m: 0.9820 - recall_m: 0.9759
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0620 - accuracy: 0.9741 - f1_m: 0.9796 - precision_m: 0.9824 - recall_m: 0.9770
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0614 - accuracy: 0.9740 - f1_m: 0.9795 - precision_m: 0.9823 - recall_m: 0.9769
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0617 - accuracy: 0.9739 - f1_m: 0.9794 - precision_m: 0.9821 - recall_m: 0.9771
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0609 - accuracy: 0.9739 - f1_m: 0.9794 - precision_m: 0.9824 - recall_m: 0.9766
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0601 - accuracy: 0.9746 - f1_m: 0.9800 - precision_m: 0.9828 - recall_m: 0.9775
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0593 - accuracy: 0.9746 - f1_m: 0.9800 - precision_m: 0.9830 - recall_m: 0.9772
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0589 - accuracy: 0.9751 - f1_m: 0.9803 - precision_m: 0.9831 - recall_m: 0.9778
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0586 - accuracy: 0.9752 - f1_m: 0.9804 - precision_m: 0.9832 - recall_m: 0.9779
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0584 - accuracy: 0.9751 - f1_m: 0.9804 - precision_m: 0.9833 - recall_m: 0.9778
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0575 - accuracy: 0.9757 - f1_m: 0.9809 - precision_m: 0.9837 - recall_m: 0.9783
Epoch 48/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0576 - accuracy: 0.9758 - f1_m: 0.9809 - precision_m: 0.9837 - recall_m: 0.9784
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0571 - accuracy: 0.9758 - f1_m: 0.9809 - precision_m: 0.9840 - recall_m: 0.9781
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0557 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9840 - recall_m: 0.9790
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0558 - accuracy: 0.9767 - f1_m: 0.9816 - precision_m: 0.9845 - recall_m: 0.9791
Epoch 52/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0552 - accuracy: 0.9768 - f1_m: 0.9818 - precision_m: 0.9846 - recall_m: 0.9792
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0548 - accuracy: 0.9767 - f1_m: 0.9816 - precision_m: 0.9847 - recall_m: 0.9789
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0543 - accuracy: 0.9772 - f1_m: 0.9820 - precision_m: 0.9850 - recall_m: 0.9792
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0540 - accuracy: 0.9773 - f1_m: 0.9821 - precision_m: 0.9847 - recall_m: 0.9798
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0539 - accuracy: 0.9773 - f1_m: 0.9821 - precision_m: 0.9852 - recall_m: 0.9794
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0529 - accuracy: 0.9775 - f1_m: 0.9823 - precision_m: 0.9849 - recall_m: 0.9800
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0528 - accuracy: 0.9775 - f1_m: 0.9823 - precision_m: 0.9852 - recall_m: 0.9797
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0525 - accuracy: 0.9777 - f1_m: 0.9824 - precision_m: 0.9853 - recall_m: 0.9797
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0523 - accuracy: 0.9779 - f1_m: 0.9826 - precision_m: 0.9852 - recall_m: 0.9802
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0514 - accuracy: 0.9785 - f1_m: 0.9830 - precision_m: 0.9855 - recall_m: 0.9808
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0514 - accuracy: 0.9786 - f1_m: 0.9831 - precision_m: 0.9861 - recall_m: 0.9803
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0519 - accuracy: 0.9779 - f1_m: 0.9825 - precision_m: 0.9852 - recall_m: 0.9801
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0499 - accuracy: 0.9789 - f1_m: 0.9834 - precision_m: 0.9860 - recall_m: 0.9811
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0504 - accuracy: 0.9788 - f1_m: 0.9833 - precision_m: 0.9857 - recall_m: 0.9811
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0501 - accuracy: 0.9790 - f1_m: 0.9834 - precision_m: 0.9859 - recall_m: 0.9811
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0489 - accuracy: 0.9796 - f1_m: 0.9839 - precision_m: 0.9865 - recall_m: 0.9816
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0499 - accuracy: 0.9792 - f1_m: 0.9836 - precision_m: 0.9861 - recall_m: 0.9813
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0494 - accuracy: 0.9791 - f1_m: 0.9835 - precision_m: 0.9863 - recall_m: 0.9809
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0487 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9865 - recall_m: 0.9817
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0485 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9866 - recall_m: 0.9816
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9800 - f1_m: 0.9842 - precision_m: 0.9868 - recall_m: 0.9817
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9798 - f1_m: 0.9841 - precision_m: 0.9864 - recall_m: 0.9819
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0467 - accuracy: 0.9802 - f1_m: 0.9844 - precision_m: 0.9869 - recall_m: 0.9822
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0473 - accuracy: 0.9803 - f1_m: 0.9845 - precision_m: 0.9869 - recall_m: 0.9822
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9799 - f1_m: 0.9841 - precision_m: 0.9867 - recall_m: 0.9819
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0469 - accuracy: 0.9803 - f1_m: 0.9845 - precision_m: 0.9869 - recall_m: 0.9823
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0450 - accuracy: 0.9814 - f1_m: 0.9854 - precision_m: 0.9877 - recall_m: 0.9833
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9806 - f1_m: 0.9847 - precision_m: 0.9873 - recall_m: 0.9824
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0458 - accuracy: 0.9807 - f1_m: 0.9848 - precision_m: 0.9872 - recall_m: 0.9826
Epoch 81/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0449 - accuracy: 0.9809 - f1_m: 0.9849 - precision_m: 0.9872 - recall_m: 0.9828
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0461 - accuracy: 0.9807 - f1_m: 0.9848 - precision_m: 0.9870 - recall_m: 0.9828
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0439 - accuracy: 0.9816 - f1_m: 0.9855 - precision_m: 0.9880 - recall_m: 0.9832
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9811 - f1_m: 0.9851 - precision_m: 0.9875 - recall_m: 0.9829
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0444 - accuracy: 0.9812 - f1_m: 0.9852 - precision_m: 0.9874 - recall_m: 0.9832
Epoch 86/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0435 - accuracy: 0.9819 - f1_m: 0.9857 - precision_m: 0.9877 - recall_m: 0.9839
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0429 - accuracy: 0.9818 - f1_m: 0.9856 - precision_m: 0.9876 - recall_m: 0.9839
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0440 - accuracy: 0.9815 - f1_m: 0.9854 - precision_m: 0.9877 - recall_m: 0.9833
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0436 - accuracy: 0.9818 - f1_m: 0.9856 - precision_m: 0.9880 - recall_m: 0.9834
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0429 - accuracy: 0.9821 - f1_m: 0.9859 - precision_m: 0.9880 - recall_m: 0.9840
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0434 - accuracy: 0.9818 - f1_m: 0.9856 - precision_m: 0.9881 - recall_m: 0.9834
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0419 - accuracy: 0.9825 - f1_m: 0.9862 - precision_m: 0.9882 - recall_m: 0.9843
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0416 - accuracy: 0.9826 - f1_m: 0.9863 - precision_m: 0.9885 - recall_m: 0.9843
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0430 - accuracy: 0.9822 - f1_m: 0.9860 - precision_m: 0.9881 - recall_m: 0.9841
Epoch 95/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0416 - accuracy: 0.9824 - f1_m: 0.9862 - precision_m: 0.9882 - recall_m: 0.9843
Epoch 96/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0427 - accuracy: 0.9825 - f1_m: 0.9862 - precision_m: 0.9882 - recall_m: 0.9844
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0416 - accuracy: 0.9826 - f1_m: 0.9863 - precision_m: 0.9884 - recall_m: 0.9844
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0407 - accuracy: 0.9831 - f1_m: 0.9867 - precision_m: 0.9888 - recall_m: 0.9848
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0412 - accuracy: 0.9827 - f1_m: 0.9864 - precision_m: 0.9884 - recall_m: 0.9846
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0400 - accuracy: 0.9833 - f1_m: 0.9868 - precision_m: 0.9888 - recall_m: 0.9850
2658/2658 [==============================] - 2s 907us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0815 - accuracy: 0.9713 - f1_m: 0.9768 - precision_m: 0.9875 - recall_m: 0.9674
score on test: 0.9713388681411743
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0382 - accuracy: 0.9838 - f1_m: 0.9869 - precision_m: 0.9945 - recall_m: 0.9801
score on train: 0.9837745428085327

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod5, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod5, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['5']=acc

score on test: 0.770369760549466
score on train: 0.8269130510340034
              precision    recall  f1-score   support

           0       0.90      0.94      0.92     30772
           1       0.96      0.81      0.88     19423
           2       0.59      0.83      0.69     14709
           3       0.69      0.59      0.63      7921
           4       0.43      0.13      0.20      5357
           5       0.70      0.72      0.71      4578
           6       0.67      0.02      0.04       909
           7       0.00      0.02      0.01       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.77     85028
   macro avg       0.50      0.41      0.41     85028
weighted avg       0.79      0.77      0.76     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(dtree1, X_train_mod5, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod5, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod5, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['5']=acc

<ipython-input-87-adb9130efb5e>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod5, y_multi_train)

score on test: 0.7773321729312697
score on train: 0.8182355326420668
              precision    recall  f1-score   support

           0       0.76      0.97      0.85     30772
           1       1.00      0.96      0.98     19423
           2       0.61      0.85      0.71     14709
           3       0.72      0.43      0.54      7921
           4       0.72      0.01      0.02      5357
           5       0.90      0.34      0.50      4578
           6       1.00      0.00      0.00       909
           7       1.00      0.00      0.01       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.78     85028
   macro avg       0.67      0.36      0.36     85028
weighted avg       0.79      0.78      0.73     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod5, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod5, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod5, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['5']=acc

score on test: 0.8534129933668909
score on train: 0.9074378729073741
              precision    recall  f1-score   support

           0       0.95      0.97      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.67      0.80      0.73     14709
           3       0.82      0.76      0.79      7921
           4       0.40      0.38      0.39      5357
           5       0.89      0.76      0.82      4578
           6       0.61      0.11      0.19       909
           7       0.60      0.07      0.12       780
           8       0.73      0.30      0.42       513
           9       0.73      0.24      0.36        66

    accuracy                           0.85     85028
   macro avg       0.74      0.54      0.58     85028
weighted avg       0.85      0.85      0.85     85028

plot_ROC_curve(xgb2, X_train_mod5, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod5,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
scores=model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['5']=scores[1]
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod5,y_multi_train)[1]))
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_9"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_36 (Dense)            (None, 128)               5504      
                                                                 
 dense_37 (Dense)            (None, 64)                8256      
                                                                 
 dense_38 (Dense)            (None, 64)                4160      
                                                                 
 dense_39 (Dense)            (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.9108 - accuracy: 0.6693 - f1_m: 1.7637 - precision_m: 1.8917 - recall_m: 1.7088
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.7095 - accuracy: 0.7365 - f1_m: 1.5835 - precision_m: 1.5281 - recall_m: 1.6525
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6288 - accuracy: 0.7644 - f1_m: 1.4801 - precision_m: 1.3776 - recall_m: 1.6072
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5841 - accuracy: 0.7812 - f1_m: 1.4179 - precision_m: 1.2975 - recall_m: 1.5700
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5572 - accuracy: 0.7920 - f1_m: 1.3757 - precision_m: 1.2491 - recall_m: 1.5379
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5402 - accuracy: 0.7966 - f1_m: 1.3466 - precision_m: 1.2148 - recall_m: 1.5178
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5263 - accuracy: 0.8007 - f1_m: 1.3321 - precision_m: 1.1968 - recall_m: 1.5086
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5154 - accuracy: 0.8049 - f1_m: 1.3180 - precision_m: 1.1807 - recall_m: 1.4980
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5061 - accuracy: 0.8088 - f1_m: 1.3018 - precision_m: 1.1625 - recall_m: 1.4858
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4959 - accuracy: 0.8111 - f1_m: 1.2957 - precision_m: 1.1558 - recall_m: 1.4806
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4861 - accuracy: 0.8159 - f1_m: 1.2830 - precision_m: 1.1419 - recall_m: 1.4703
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4754 - accuracy: 0.8198 - f1_m: 1.2706 - precision_m: 1.1284 - recall_m: 1.4603
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4671 - accuracy: 0.8231 - f1_m: 1.2669 - precision_m: 1.1242 - recall_m: 1.4575
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4596 - accuracy: 0.8255 - f1_m: 1.2599 - precision_m: 1.1166 - recall_m: 1.4523
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4532 - accuracy: 0.8283 - f1_m: 1.2517 - precision_m: 1.1066 - recall_m: 1.4470
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4485 - accuracy: 0.8300 - f1_m: 1.2484 - precision_m: 1.1026 - recall_m: 1.4449
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4447 - accuracy: 0.8302 - f1_m: 1.2438 - precision_m: 1.0980 - recall_m: 1.4406
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4409 - accuracy: 0.8312 - f1_m: 1.2410 - precision_m: 1.0945 - recall_m: 1.4390
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4388 - accuracy: 0.8326 - f1_m: 1.2427 - precision_m: 1.0960 - recall_m: 1.4407
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4351 - accuracy: 0.8338 - f1_m: 1.2384 - precision_m: 1.0915 - recall_m: 1.4374
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4315 - accuracy: 0.8354 - f1_m: 1.2369 - precision_m: 1.0897 - recall_m: 1.4364
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4287 - accuracy: 0.8352 - f1_m: 1.2348 - precision_m: 1.0861 - recall_m: 1.4362
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4278 - accuracy: 0.8360 - f1_m: 1.2312 - precision_m: 1.0826 - recall_m: 1.4329
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4247 - accuracy: 0.8374 - f1_m: 1.2327 - precision_m: 1.0847 - recall_m: 1.4336
Epoch 25/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4219 - accuracy: 0.8376 - f1_m: 1.2320 - precision_m: 1.0834 - recall_m: 1.4339
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4207 - accuracy: 0.8382 - f1_m: 1.2309 - precision_m: 1.0817 - recall_m: 1.4332
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4193 - accuracy: 0.8394 - f1_m: 1.2295 - precision_m: 1.0804 - recall_m: 1.4322
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4179 - accuracy: 0.8395 - f1_m: 1.2278 - precision_m: 1.0788 - recall_m: 1.4306
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4151 - accuracy: 0.8402 - f1_m: 1.2291 - precision_m: 1.0802 - recall_m: 1.4319
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4138 - accuracy: 0.8407 - f1_m: 1.2265 - precision_m: 1.0770 - recall_m: 1.4299
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4125 - accuracy: 0.8411 - f1_m: 1.2259 - precision_m: 1.0767 - recall_m: 1.4287
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4118 - accuracy: 0.8413 - f1_m: 1.2240 - precision_m: 1.0745 - recall_m: 1.4276
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4107 - accuracy: 0.8414 - f1_m: 1.2222 - precision_m: 1.0727 - recall_m: 1.4260
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4100 - accuracy: 0.8420 - f1_m: 1.2250 - precision_m: 1.0756 - recall_m: 1.4286
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4066 - accuracy: 0.8420 - f1_m: 1.2222 - precision_m: 1.0721 - recall_m: 1.4268
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4061 - accuracy: 0.8435 - f1_m: 1.2231 - precision_m: 1.0720 - recall_m: 1.4289
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4040 - accuracy: 0.8439 - f1_m: 1.2201 - precision_m: 1.0700 - recall_m: 1.4252
Epoch 38/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.4041 - accuracy: 0.8431 - f1_m: 1.2198 - precision_m: 1.0685 - recall_m: 1.4265
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4025 - accuracy: 0.8441 - f1_m: 1.2188 - precision_m: 1.0683 - recall_m: 1.4246
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4006 - accuracy: 0.8450 - f1_m: 1.2189 - precision_m: 1.0683 - recall_m: 1.4244
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4000 - accuracy: 0.8447 - f1_m: 1.2144 - precision_m: 1.0643 - recall_m: 1.4203
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3993 - accuracy: 0.8449 - f1_m: 1.2185 - precision_m: 1.0686 - recall_m: 1.4232
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3986 - accuracy: 0.8449 - f1_m: 1.2156 - precision_m: 1.0644 - recall_m: 1.4226
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3965 - accuracy: 0.8459 - f1_m: 1.2155 - precision_m: 1.0648 - recall_m: 1.4217
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3957 - accuracy: 0.8466 - f1_m: 1.2135 - precision_m: 1.0620 - recall_m: 1.4213
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3948 - accuracy: 0.8465 - f1_m: 1.2124 - precision_m: 1.0610 - recall_m: 1.4197
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3940 - accuracy: 0.8472 - f1_m: 1.2126 - precision_m: 1.0617 - recall_m: 1.4196
Epoch 48/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3941 - accuracy: 0.8469 - f1_m: 1.2126 - precision_m: 1.0611 - recall_m: 1.4203
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3922 - accuracy: 0.8477 - f1_m: 1.2102 - precision_m: 1.0593 - recall_m: 1.4175
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3911 - accuracy: 0.8475 - f1_m: 1.2112 - precision_m: 1.0595 - recall_m: 1.4196
Epoch 51/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3905 - accuracy: 0.8472 - f1_m: 1.2126 - precision_m: 1.0615 - recall_m: 1.4196
Epoch 52/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3897 - accuracy: 0.8481 - f1_m: 1.2104 - precision_m: 1.0591 - recall_m: 1.4183
Epoch 53/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3898 - accuracy: 0.8484 - f1_m: 1.2096 - precision_m: 1.0578 - recall_m: 1.4176
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3874 - accuracy: 0.8484 - f1_m: 1.2092 - precision_m: 1.0575 - recall_m: 1.4173
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3879 - accuracy: 0.8483 - f1_m: 1.2048 - precision_m: 1.0525 - recall_m: 1.4143
Epoch 56/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3865 - accuracy: 0.8493 - f1_m: 1.2077 - precision_m: 1.0557 - recall_m: 1.4164
Epoch 57/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3861 - accuracy: 0.8492 - f1_m: 1.2071 - precision_m: 1.0551 - recall_m: 1.4163
Epoch 58/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3849 - accuracy: 0.8498 - f1_m: 1.2076 - precision_m: 1.0560 - recall_m: 1.4157
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3856 - accuracy: 0.8499 - f1_m: 1.2067 - precision_m: 1.0548 - recall_m: 1.4156
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3845 - accuracy: 0.8502 - f1_m: 1.2045 - precision_m: 1.0528 - recall_m: 1.4134
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3829 - accuracy: 0.8504 - f1_m: 1.2065 - precision_m: 1.0545 - recall_m: 1.4154
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3811 - accuracy: 0.8512 - f1_m: 1.2041 - precision_m: 1.0521 - recall_m: 1.4131
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3821 - accuracy: 0.8507 - f1_m: 1.2026 - precision_m: 1.0501 - recall_m: 1.4128
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3805 - accuracy: 0.8510 - f1_m: 1.2059 - precision_m: 1.0548 - recall_m: 1.4137
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3821 - accuracy: 0.8508 - f1_m: 1.2007 - precision_m: 1.0485 - recall_m: 1.4102
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3808 - accuracy: 0.8510 - f1_m: 1.2011 - precision_m: 1.0486 - recall_m: 1.4112
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3792 - accuracy: 0.8517 - f1_m: 1.2033 - precision_m: 1.0509 - recall_m: 1.4133
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3787 - accuracy: 0.8521 - f1_m: 1.2028 - precision_m: 1.0501 - recall_m: 1.4132
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3782 - accuracy: 0.8519 - f1_m: 1.2016 - precision_m: 1.0489 - recall_m: 1.4117
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3784 - accuracy: 0.8519 - f1_m: 1.2035 - precision_m: 1.0511 - recall_m: 1.4129
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3763 - accuracy: 0.8522 - f1_m: 1.1980 - precision_m: 1.0450 - recall_m: 1.4092
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3774 - accuracy: 0.8525 - f1_m: 1.2017 - precision_m: 1.0489 - recall_m: 1.4119
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3761 - accuracy: 0.8530 - f1_m: 1.2002 - precision_m: 1.0479 - recall_m: 1.4102
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3753 - accuracy: 0.8527 - f1_m: 1.1991 - precision_m: 1.0459 - recall_m: 1.4105
Epoch 75/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.3735 - accuracy: 0.8534 - f1_m: 1.1986 - precision_m: 1.0458 - recall_m: 1.4097
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3741 - accuracy: 0.8533 - f1_m: 1.1974 - precision_m: 1.0441 - recall_m: 1.4095
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3730 - accuracy: 0.8538 - f1_m: 1.1993 - precision_m: 1.0463 - recall_m: 1.4103
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3728 - accuracy: 0.8541 - f1_m: 1.1979 - precision_m: 1.0451 - recall_m: 1.4092
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3723 - accuracy: 0.8540 - f1_m: 1.2006 - precision_m: 1.0475 - recall_m: 1.4118
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3732 - accuracy: 0.8541 - f1_m: 1.1997 - precision_m: 1.0469 - recall_m: 1.4104
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3728 - accuracy: 0.8544 - f1_m: 1.1973 - precision_m: 1.0442 - recall_m: 1.4088
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3709 - accuracy: 0.8543 - f1_m: 1.1960 - precision_m: 1.0427 - recall_m: 1.4078
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3710 - accuracy: 0.8540 - f1_m: 1.1985 - precision_m: 1.0453 - recall_m: 1.4101
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3704 - accuracy: 0.8552 - f1_m: 1.1977 - precision_m: 1.0440 - recall_m: 1.4098
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3708 - accuracy: 0.8544 - f1_m: 1.1957 - precision_m: 1.0421 - recall_m: 1.4078
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3698 - accuracy: 0.8550 - f1_m: 1.1952 - precision_m: 1.0417 - recall_m: 1.4073
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3689 - accuracy: 0.8552 - f1_m: 1.1954 - precision_m: 1.0418 - recall_m: 1.4080
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3684 - accuracy: 0.8561 - f1_m: 1.1962 - precision_m: 1.0427 - recall_m: 1.4085
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3695 - accuracy: 0.8546 - f1_m: 1.1956 - precision_m: 1.0432 - recall_m: 1.4059
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3666 - accuracy: 0.8559 - f1_m: 1.1942 - precision_m: 1.0404 - recall_m: 1.4072
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3653 - accuracy: 0.8565 - f1_m: 1.1957 - precision_m: 1.0422 - recall_m: 1.4079
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3673 - accuracy: 0.8554 - f1_m: 1.1957 - precision_m: 1.0418 - recall_m: 1.4084
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3684 - accuracy: 0.8558 - f1_m: 1.1936 - precision_m: 1.0396 - recall_m: 1.4064
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3656 - accuracy: 0.8560 - f1_m: 1.1941 - precision_m: 1.0403 - recall_m: 1.4071
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3658 - accuracy: 0.8568 - f1_m: 1.1927 - precision_m: 1.0387 - recall_m: 1.4061
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3653 - accuracy: 0.8569 - f1_m: 1.1931 - precision_m: 1.0393 - recall_m: 1.4062
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3642 - accuracy: 0.8563 - f1_m: 1.1936 - precision_m: 1.0393 - recall_m: 1.4069
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3645 - accuracy: 0.8566 - f1_m: 1.1913 - precision_m: 1.0374 - recall_m: 1.4046
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3636 - accuracy: 0.8568 - f1_m: 1.1900 - precision_m: 1.0357 - recall_m: 1.4037
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3636 - accuracy: 0.8567 - f1_m: 1.1921 - precision_m: 1.0380 - recall_m: 1.4056
2658/2658 [==============================] - 3s 924us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4225 - accuracy: 0.8457 - f1_m: 1.2004 - precision_m: 1.0583 - recall_m: 1.4088
score on test: 0.8456978797912598
5395/5395 [==============================] - 7s 1ms/step - loss: 0.3499 - accuracy: 0.8604 - f1_m: 1.2004 - precision_m: 1.0582 - recall_m: 1.4094
score on train: 0.8603545427322388

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.7599266831939349

Noise range 3

noise6 = 6
X_train_mod6 = modify_dataset(X_train, variables_to_poison, noise6)

X_train_mod6

array([[ 1.79214449, -0.2082886 ,  3.79655808, ...,  0.89924558,
        -0.3952588 , -0.12070382],
       [-0.57638416,  1.78686151, -3.32366538, ..., -2.90888974,
         7.8279375 , -0.12070382],
       [-1.0569863 , -0.20828827,  3.46781103, ...,  9.6926833 ,
         3.01798364, -0.12070382],
       ...,
       [-0.47187522, -0.06694436, -0.24042675, ...,  9.16582496,
         3.1901775 , -0.12070382],
       [-0.1666061 , -0.18783648, -0.96993323, ..., -0.44809629,
         7.74497192, -0.12070382],
       [-0.6757627 , -0.20011705, -6.60339241, ..., -2.33324492,
         0.82474916, -0.12070382]])

from sklearn import metrics
from sklearn.metrics import (confusion_matrix, accuracy_score, precision_score, recall_score, f1_score)
lr1=LogisticRegression()
lr1.fit(X_train_mod6, y_train)
y_pred = lr1.predict(X_test)
print("score on test: " + str(lr1.score(X_test, y_test)))
print("score on train: "+ str(lr1.score(X_train_mod6, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_log['6']=acc

/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py:1143: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

score on test: 0.8916239356447287
score on train: 0.8925968835080809
              precision    recall  f1-score   support

           0       0.93      0.75      0.83     30772
           1       0.87      0.97      0.92     54256

    accuracy                           0.89     85028
   macro avg       0.90      0.86      0.88     85028
weighted avg       0.90      0.89      0.89     85028

# ROC curve and AUC
lr1_predproba = lr1.predict_proba(X_test)[:, 1]
AUC_lr1 = roc_auc_score(y_test, lr1_predproba)
print(f'AUC: {AUC_lr1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, lr1_predproba)
plt.plot(fpr, tpr, label='logistic regression Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9529

rf1 = RandomForestClassifier(n_estimators=30, max_depth=10)
rf1.fit(X_train_mod6, y_train)
y_pred = rf1.predict(X_test)
print("score on test: " + str(rf1.score(X_test, y_test)))
print("score on train: "+ str(rf1.score(X_train_mod6, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_rf_b['6']=acc

<ipython-input-96-6f5d5b3d70e0>:2: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf1.fit(X_train_mod6, y_train)

score on test: 0.8954579667874112
score on train: 0.952302612523895
              precision    recall  f1-score   support

           0       0.81      0.92      0.86     30772
           1       0.95      0.88      0.91     54256

    accuracy                           0.90     85028
   macro avg       0.88      0.90      0.89     85028
weighted avg       0.90      0.90      0.90     85028

rf1_predproba = rf1.predict_proba(X_test)[:, 1]
AUC_rf1 = roc_auc_score(y_test, rf1_predproba)
print(f'AUC: {AUC_rf1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, rf1_predproba)
plt.plot(fpr, tpr, label='random forest Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9722

xgb1 = xgb.XGBClassifier(objective="binary:logistic", random_state=42)
xgb1.fit(X_train_mod6, y_train)
y_pred = xgb1.predict(X_test)
print("score on test: " + str(xgb1.score(X_test, y_test)))
print("score on train: "+ str(xgb1.score(X_train_mod6, y_train)))
print(metrics.classification_report(y_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_test, y_pred)
confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()
res_xgb_b['6']=acc

score on test: 0.9674577786141035
score on train: 0.9924462723744425
              precision    recall  f1-score   support

           0       0.96      0.95      0.95     30772
           1       0.97      0.98      0.97     54256

    accuracy                           0.97     85028
   macro avg       0.97      0.96      0.96     85028
weighted avg       0.97      0.97      0.97     85028

# ROC curve and AUC
xgb1_predproba = xgb1.predict_proba(X_test)[:, 1]
AUC_xgb1 = roc_auc_score(y_test, xgb1_predproba)
print(f'AUC: {AUC_xgb1:.4f}')
# Compute false positive rate, true positive rate, and thresholds
fpr, tpr, thresholds = roc_curve(y_test, xgb1_predproba)
plt.plot(fpr, tpr, label='Gradient boosting Model')
plt.plot([0, 1], [0, 1], 'k--', label='Random classification')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve')
plt.legend()
plt.show()

AUC: 0.9952

model=models.Sequential()
model.add(layers.Dense(128,activation='relu',input_shape=(42,)))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(64,activation='relu'))
model.add(layers.Dense(1,activation='sigmoid'))
model.summary

model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy', f1_m, precision_m, recall_m])

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)
model.fit(X_train_mod6, y_train, batch_size=128, epochs=100, callbacks=[earlystopping])

y_pred = model.predict(X_test)
print("score on test: " + str(model.evaluate(X_test,y_test)[1]))
print("score on train: "+ str(model.evaluate(X_train_mod6,y_train)[1]))
loss, accuracy, f1_score, precision, recall = model.evaluate(X_test, y_test, verbose=0)
scores=model.evaluate(X_test, y_test, verbose=0)
res_nn_b['6']=scores[1]
from sklearn import metrics
y_pred_mat = y_pred.round()
confusion_matrix = metrics.confusion_matrix(y_test, y_pred_mat)
cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = [False, True])
cm_display.plot()
plt.show()

Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.2754 - accuracy: 0.8759 - f1_m: 0.9068 - precision_m: 0.8720 - recall_m: 0.9463
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.2097 - accuracy: 0.9037 - f1_m: 0.9265 - precision_m: 0.9008 - recall_m: 0.9550
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1872 - accuracy: 0.9137 - f1_m: 0.9334 - precision_m: 0.9174 - recall_m: 0.9509
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1738 - accuracy: 0.9196 - f1_m: 0.9377 - precision_m: 0.9259 - recall_m: 0.9508
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1596 - accuracy: 0.9257 - f1_m: 0.9421 - precision_m: 0.9346 - recall_m: 0.9506
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1484 - accuracy: 0.9313 - f1_m: 0.9463 - precision_m: 0.9421 - recall_m: 0.9512
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1399 - accuracy: 0.9366 - f1_m: 0.9503 - precision_m: 0.9475 - recall_m: 0.9538
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1249 - accuracy: 0.9447 - f1_m: 0.9565 - precision_m: 0.9559 - recall_m: 0.9577
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1090 - accuracy: 0.9534 - f1_m: 0.9634 - precision_m: 0.9634 - recall_m: 0.9639
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.1011 - accuracy: 0.9569 - f1_m: 0.9661 - precision_m: 0.9673 - recall_m: 0.9654
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0956 - accuracy: 0.9591 - f1_m: 0.9678 - precision_m: 0.9691 - recall_m: 0.9670
Epoch 12/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0928 - accuracy: 0.9605 - f1_m: 0.9689 - precision_m: 0.9705 - recall_m: 0.9678
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0899 - accuracy: 0.9618 - f1_m: 0.9699 - precision_m: 0.9714 - recall_m: 0.9688
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0877 - accuracy: 0.9628 - f1_m: 0.9707 - precision_m: 0.9724 - recall_m: 0.9693
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0854 - accuracy: 0.9633 - f1_m: 0.9711 - precision_m: 0.9735 - recall_m: 0.9691
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0847 - accuracy: 0.9640 - f1_m: 0.9716 - precision_m: 0.9740 - recall_m: 0.9697
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0835 - accuracy: 0.9645 - f1_m: 0.9720 - precision_m: 0.9744 - recall_m: 0.9700
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0818 - accuracy: 0.9654 - f1_m: 0.9727 - precision_m: 0.9754 - recall_m: 0.9704
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0802 - accuracy: 0.9659 - f1_m: 0.9732 - precision_m: 0.9758 - recall_m: 0.9709
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0791 - accuracy: 0.9663 - f1_m: 0.9734 - precision_m: 0.9763 - recall_m: 0.9709
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0784 - accuracy: 0.9667 - f1_m: 0.9737 - precision_m: 0.9768 - recall_m: 0.9709
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0768 - accuracy: 0.9668 - f1_m: 0.9739 - precision_m: 0.9770 - recall_m: 0.9711
Epoch 23/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0757 - accuracy: 0.9677 - f1_m: 0.9746 - precision_m: 0.9779 - recall_m: 0.9716
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0749 - accuracy: 0.9684 - f1_m: 0.9751 - precision_m: 0.9783 - recall_m: 0.9723
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0738 - accuracy: 0.9685 - f1_m: 0.9752 - precision_m: 0.9784 - recall_m: 0.9723
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0725 - accuracy: 0.9693 - f1_m: 0.9757 - precision_m: 0.9794 - recall_m: 0.9724
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0730 - accuracy: 0.9689 - f1_m: 0.9754 - precision_m: 0.9789 - recall_m: 0.9723
Epoch 28/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.0720 - accuracy: 0.9693 - f1_m: 0.9759 - precision_m: 0.9793 - recall_m: 0.9728
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0717 - accuracy: 0.9695 - f1_m: 0.9759 - precision_m: 0.9796 - recall_m: 0.9726
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0702 - accuracy: 0.9703 - f1_m: 0.9766 - precision_m: 0.9799 - recall_m: 0.9736
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0699 - accuracy: 0.9701 - f1_m: 0.9764 - precision_m: 0.9799 - recall_m: 0.9733
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0687 - accuracy: 0.9709 - f1_m: 0.9771 - precision_m: 0.9807 - recall_m: 0.9738
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0689 - accuracy: 0.9706 - f1_m: 0.9768 - precision_m: 0.9803 - recall_m: 0.9736
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0679 - accuracy: 0.9707 - f1_m: 0.9768 - precision_m: 0.9805 - recall_m: 0.9735
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0671 - accuracy: 0.9714 - f1_m: 0.9775 - precision_m: 0.9809 - recall_m: 0.9743
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0669 - accuracy: 0.9715 - f1_m: 0.9776 - precision_m: 0.9810 - recall_m: 0.9744
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0656 - accuracy: 0.9721 - f1_m: 0.9779 - precision_m: 0.9816 - recall_m: 0.9747
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0648 - accuracy: 0.9725 - f1_m: 0.9783 - precision_m: 0.9817 - recall_m: 0.9751
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0658 - accuracy: 0.9724 - f1_m: 0.9782 - precision_m: 0.9817 - recall_m: 0.9751
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0639 - accuracy: 0.9728 - f1_m: 0.9785 - precision_m: 0.9819 - recall_m: 0.9754
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0640 - accuracy: 0.9728 - f1_m: 0.9785 - precision_m: 0.9819 - recall_m: 0.9755
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0628 - accuracy: 0.9732 - f1_m: 0.9789 - precision_m: 0.9824 - recall_m: 0.9756
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0633 - accuracy: 0.9732 - f1_m: 0.9789 - precision_m: 0.9823 - recall_m: 0.9758
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0620 - accuracy: 0.9731 - f1_m: 0.9788 - precision_m: 0.9821 - recall_m: 0.9757
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0620 - accuracy: 0.9733 - f1_m: 0.9790 - precision_m: 0.9825 - recall_m: 0.9759
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0615 - accuracy: 0.9738 - f1_m: 0.9793 - precision_m: 0.9829 - recall_m: 0.9760
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0611 - accuracy: 0.9741 - f1_m: 0.9796 - precision_m: 0.9828 - recall_m: 0.9766
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0605 - accuracy: 0.9745 - f1_m: 0.9799 - precision_m: 0.9833 - recall_m: 0.9768
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0597 - accuracy: 0.9745 - f1_m: 0.9799 - precision_m: 0.9834 - recall_m: 0.9767
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0598 - accuracy: 0.9746 - f1_m: 0.9800 - precision_m: 0.9832 - recall_m: 0.9770
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0590 - accuracy: 0.9748 - f1_m: 0.9801 - precision_m: 0.9833 - recall_m: 0.9771
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0594 - accuracy: 0.9747 - f1_m: 0.9801 - precision_m: 0.9833 - recall_m: 0.9772
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0576 - accuracy: 0.9755 - f1_m: 0.9807 - precision_m: 0.9841 - recall_m: 0.9776
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0578 - accuracy: 0.9754 - f1_m: 0.9806 - precision_m: 0.9841 - recall_m: 0.9774
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0575 - accuracy: 0.9756 - f1_m: 0.9807 - precision_m: 0.9841 - recall_m: 0.9777
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0578 - accuracy: 0.9757 - f1_m: 0.9808 - precision_m: 0.9844 - recall_m: 0.9775
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0560 - accuracy: 0.9762 - f1_m: 0.9812 - precision_m: 0.9847 - recall_m: 0.9781
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0564 - accuracy: 0.9759 - f1_m: 0.9809 - precision_m: 0.9843 - recall_m: 0.9778
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0567 - accuracy: 0.9759 - f1_m: 0.9810 - precision_m: 0.9841 - recall_m: 0.9782
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0557 - accuracy: 0.9763 - f1_m: 0.9813 - precision_m: 0.9850 - recall_m: 0.9778
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0551 - accuracy: 0.9765 - f1_m: 0.9814 - precision_m: 0.9847 - recall_m: 0.9784
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0553 - accuracy: 0.9764 - f1_m: 0.9814 - precision_m: 0.9849 - recall_m: 0.9782
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0549 - accuracy: 0.9767 - f1_m: 0.9816 - precision_m: 0.9846 - recall_m: 0.9789
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0547 - accuracy: 0.9770 - f1_m: 0.9818 - precision_m: 0.9852 - recall_m: 0.9787
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0530 - accuracy: 0.9774 - f1_m: 0.9822 - precision_m: 0.9854 - recall_m: 0.9792
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0538 - accuracy: 0.9770 - f1_m: 0.9818 - precision_m: 0.9852 - recall_m: 0.9788
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0532 - accuracy: 0.9774 - f1_m: 0.9822 - precision_m: 0.9855 - recall_m: 0.9791
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0531 - accuracy: 0.9777 - f1_m: 0.9824 - precision_m: 0.9857 - recall_m: 0.9792
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0528 - accuracy: 0.9774 - f1_m: 0.9822 - precision_m: 0.9855 - recall_m: 0.9791
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0525 - accuracy: 0.9777 - f1_m: 0.9824 - precision_m: 0.9857 - recall_m: 0.9794
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0527 - accuracy: 0.9778 - f1_m: 0.9825 - precision_m: 0.9858 - recall_m: 0.9794
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0520 - accuracy: 0.9779 - f1_m: 0.9825 - precision_m: 0.9858 - recall_m: 0.9795
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0511 - accuracy: 0.9784 - f1_m: 0.9830 - precision_m: 0.9859 - recall_m: 0.9803
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0515 - accuracy: 0.9783 - f1_m: 0.9829 - precision_m: 0.9860 - recall_m: 0.9800
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0506 - accuracy: 0.9786 - f1_m: 0.9831 - precision_m: 0.9864 - recall_m: 0.9800
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0505 - accuracy: 0.9787 - f1_m: 0.9832 - precision_m: 0.9863 - recall_m: 0.9804
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0514 - accuracy: 0.9786 - f1_m: 0.9831 - precision_m: 0.9865 - recall_m: 0.9800
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0499 - accuracy: 0.9790 - f1_m: 0.9835 - precision_m: 0.9865 - recall_m: 0.9807
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0499 - accuracy: 0.9790 - f1_m: 0.9834 - precision_m: 0.9864 - recall_m: 0.9806
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0502 - accuracy: 0.9791 - f1_m: 0.9835 - precision_m: 0.9868 - recall_m: 0.9804
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0486 - accuracy: 0.9791 - f1_m: 0.9835 - precision_m: 0.9867 - recall_m: 0.9805
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0485 - accuracy: 0.9791 - f1_m: 0.9835 - precision_m: 0.9866 - recall_m: 0.9807
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0486 - accuracy: 0.9795 - f1_m: 0.9839 - precision_m: 0.9871 - recall_m: 0.9808
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0489 - accuracy: 0.9793 - f1_m: 0.9837 - precision_m: 0.9867 - recall_m: 0.9809
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9795 - f1_m: 0.9838 - precision_m: 0.9870 - recall_m: 0.9808
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0480 - accuracy: 0.9797 - f1_m: 0.9840 - precision_m: 0.9869 - recall_m: 0.9813
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0478 - accuracy: 0.9799 - f1_m: 0.9841 - precision_m: 0.9871 - recall_m: 0.9814
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0469 - accuracy: 0.9804 - f1_m: 0.9845 - precision_m: 0.9875 - recall_m: 0.9818
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0471 - accuracy: 0.9800 - f1_m: 0.9842 - precision_m: 0.9871 - recall_m: 0.9815
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0463 - accuracy: 0.9806 - f1_m: 0.9847 - precision_m: 0.9876 - recall_m: 0.9819
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0476 - accuracy: 0.9799 - f1_m: 0.9842 - precision_m: 0.9871 - recall_m: 0.9814
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0465 - accuracy: 0.9806 - f1_m: 0.9846 - precision_m: 0.9876 - recall_m: 0.9819
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9806 - f1_m: 0.9847 - precision_m: 0.9878 - recall_m: 0.9818
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0460 - accuracy: 0.9807 - f1_m: 0.9848 - precision_m: 0.9876 - recall_m: 0.9821
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0466 - accuracy: 0.9805 - f1_m: 0.9846 - precision_m: 0.9877 - recall_m: 0.9818
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0456 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9878 - recall_m: 0.9822
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0452 - accuracy: 0.9809 - f1_m: 0.9850 - precision_m: 0.9879 - recall_m: 0.9822
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0448 - accuracy: 0.9813 - f1_m: 0.9853 - precision_m: 0.9882 - recall_m: 0.9825
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0451 - accuracy: 0.9808 - f1_m: 0.9849 - precision_m: 0.9878 - recall_m: 0.9822
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.0444 - accuracy: 0.9813 - f1_m: 0.9852 - precision_m: 0.9880 - recall_m: 0.9827
2658/2658 [==============================] - 2s 872us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.0764 - accuracy: 0.9729 - f1_m: 0.9783 - precision_m: 0.9820 - recall_m: 0.9756
score on test: 0.9729265570640564
5395/5395 [==============================] - 6s 1ms/step - loss: 0.0427 - accuracy: 0.9822 - f1_m: 0.9857 - precision_m: 0.9888 - recall_m: 0.9835
score on train: 0.9822047352790833

from sklearn.metrics import auc
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred)
auc_keras = auc(fpr_keras, tpr_keras)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()

dtree1 = DecisionTreeClassifier(max_depth = 10).fit(X_train_mod6, y_multi_train)
y_pred = dtree1.predict(X_test)
print("score on test: " + str(dtree1.score(X_test, y_multi_test)))
print("score on train: "+ str(dtree1.score(X_train_mod6, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_tree['6']=acc

score on test: 0.7619490050336359
score on train: 0.8268551236749117
              precision    recall  f1-score   support

           0       0.90      0.91      0.91     30772
           1       0.96      0.81      0.88     19423
           2       0.57      0.83      0.68     14709
           3       0.69      0.59      0.63      7921
           4       0.44      0.13      0.20      5357
           5       0.72      0.72      0.72      4578
           6       0.67      0.02      0.04       909
           7       0.00      0.02      0.01       780
           8       0.38      0.01      0.01       513
           9       0.00      0.00      0.00        66

    accuracy                           0.76     85028
   macro avg       0.53      0.40      0.41     85028
weighted avg       0.78      0.76      0.76     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(dtree1, X_train_mod6, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for DecisionTreeClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=DecisionTreeClassifier(max_depth=10))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod6, y_multi_train)
y_pred = rf2.predict(X_test)
print("score on test: " + str(rf2.score(X_test, y_multi_test)))
print("score on train: "+ str(rf2.score(X_train_mod6, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
accuracy = rf2.score(X_test, y_multi_test)
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_rf_m['6']=acc

<ipython-input-104-3b082d3dcb17>:1: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  rf2 = RandomForestClassifier(n_estimators=30, max_depth=10).fit(X_train_mod6, y_multi_train)

score on test: 0.69498282918568
score on train: 0.8142559230724671
              precision    recall  f1-score   support

           0       0.59      0.98      0.73     30772
           1       1.00      0.80      0.89     19423
           2       0.72      0.62      0.67     14709
           3       0.76      0.39      0.52      7921
           4       0.59      0.01      0.02      5357
           5       0.88      0.28      0.42      4578
           6       1.00      0.00      0.01       909
           7       1.00      0.01      0.01       780
           8       0.00      0.00      0.00       513
           9       0.00      0.00      0.00        66

    accuracy                           0.69     85028
   macro avg       0.65      0.31      0.33     85028
weighted avg       0.74      0.69      0.66     85028

/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))

plot_ROC_curve(rf2, X_train_mod6, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for RandomForestClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=RandomForestClassifier(max_depth=10, n_estimators=30))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

xgb2 = xgb.XGBClassifier(objective="multi:softprob", random_state=42)
xgb2.fit(X_train_mod6, y_multi_train)
y_pred = xgb2.predict(X_test)
print("score on test: " + str(xgb2.score(X_test, y_multi_test)))
print("score on train: "+ str(xgb2.score(X_train_mod6, y_multi_train)))
print(metrics.classification_report(y_multi_test, y_pred))
acc=sklearn.metrics.accuracy_score(y_multi_test, y_pred)
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)
res_xgb_m['6']=acc

score on test: 0.8536246883379592
score on train: 0.9073683600764642
              precision    recall  f1-score   support

           0       0.95      0.97      0.96     30772
           1       1.00      0.98      0.99     19423
           2       0.67      0.80      0.73     14709
           3       0.78      0.79      0.79      7921
           4       0.41      0.35      0.38      5357
           5       0.90      0.75      0.82      4578
           6       0.68      0.11      0.18       909
           7       0.58      0.07      0.13       780
           8       0.70      0.31      0.43       513
           9       0.50      0.03      0.06        66

    accuracy                           0.85     85028
   macro avg       0.72      0.52      0.55     85028
weighted avg       0.85      0.85      0.85     85028

plot_ROC_curve(xgb2, X_train_mod6, y_multi_train, X_test, y_multi_test)

ROCAUC(ax=<Axes: title={'center': 'ROC Curves for XGBClassifier'}, xlabel='False Positive Rate', ylabel='True Positive Rate'>,
       estimator=XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, g...
                               grow_policy=None, importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, n_estimators=100,
                               n_jobs=None, num_parallel_tree=None,
                               objective='multi:softprob', predictor=None, ...))

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

model2 = models.Sequential()
model2.add(layers.Dense(128, input_shape = (42,), activation = "relu"))
model2.add(layers.Dense(64, activation = "relu"))
model2.add(layers.Dense(64,activation="relu"))
model2.add(layers.Dense(10, activation = "softmax"))
model2.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics = ["accuracy", f1_m, precision_m, recall_m])
model2.summary()

earlystopping = callbacks.EarlyStopping(monitor="loss",
                                        mode="min", patience=5,
                                        restore_best_weights=True)

model2.fit(X_train_mod6,y_multi_train,epochs=100,batch_size=128,callbacks=[earlystopping])
y_pred_arr = model2.predict(X_test)
y_pred = []
for el in y_pred_arr:
    ind = np.argmax(el)
    y_pred.append(ind)
scores=model2.evaluate(X_test, y_multi_test, verbose=0)
res_nn_m['6']=scores[1]
print("score on test: " + str(model2.evaluate(X_test,y_multi_test)[1]))
print("score on train: "+ str(model2.evaluate(X_train_mod6,y_multi_train)[1]))
cm = confusion_matrix(y_multi_test, y_pred)
plot_confusion_matrix(cm, attack_cat)

Model: "sequential_11"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 dense_44 (Dense)            (None, 128)               5504      
                                                                 
 dense_45 (Dense)            (None, 64)                8256      
                                                                 
 dense_46 (Dense)            (None, 64)                4160      
                                                                 
 dense_47 (Dense)            (None, 10)                650       
                                                                 
=================================================================
Total params: 18,570
Trainable params: 18,570
Non-trainable params: 0
_________________________________________________________________
Epoch 1/100
1349/1349 [==============================] - 3s 2ms/step - loss: 0.9364 - accuracy: 0.6598 - f1_m: 1.7687 - precision_m: 2.2474 - recall_m: 1.7018
Epoch 2/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.7272 - accuracy: 0.7285 - f1_m: 1.6018 - precision_m: 1.5573 - recall_m: 1.6587
Epoch 3/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.6436 - accuracy: 0.7579 - f1_m: 1.5029 - precision_m: 1.4104 - recall_m: 1.6167
Epoch 4/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5936 - accuracy: 0.7778 - f1_m: 1.4321 - precision_m: 1.3154 - recall_m: 1.5793
Epoch 5/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5630 - accuracy: 0.7885 - f1_m: 1.3858 - precision_m: 1.2581 - recall_m: 1.5493
Epoch 6/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5452 - accuracy: 0.7951 - f1_m: 1.3588 - precision_m: 1.2257 - recall_m: 1.5310
Epoch 7/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5299 - accuracy: 0.8009 - f1_m: 1.3329 - precision_m: 1.1970 - recall_m: 1.5106
Epoch 8/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5172 - accuracy: 0.8050 - f1_m: 1.3184 - precision_m: 1.1800 - recall_m: 1.5004
Epoch 9/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.5037 - accuracy: 0.8102 - f1_m: 1.3017 - precision_m: 1.1598 - recall_m: 1.4894
Epoch 10/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4928 - accuracy: 0.8141 - f1_m: 1.2942 - precision_m: 1.1517 - recall_m: 1.4837
Epoch 11/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4823 - accuracy: 0.8181 - f1_m: 1.2800 - precision_m: 1.1355 - recall_m: 1.4732
Epoch 12/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4729 - accuracy: 0.8216 - f1_m: 1.2698 - precision_m: 1.1244 - recall_m: 1.4649
Epoch 13/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4652 - accuracy: 0.8248 - f1_m: 1.2624 - precision_m: 1.1156 - recall_m: 1.4608
Epoch 14/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4590 - accuracy: 0.8265 - f1_m: 1.2574 - precision_m: 1.1105 - recall_m: 1.4553
Epoch 15/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4542 - accuracy: 0.8287 - f1_m: 1.2546 - precision_m: 1.1066 - recall_m: 1.4548
Epoch 16/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4509 - accuracy: 0.8290 - f1_m: 1.2523 - precision_m: 1.1042 - recall_m: 1.4522
Epoch 17/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4474 - accuracy: 0.8308 - f1_m: 1.2466 - precision_m: 1.0973 - recall_m: 1.4492
Epoch 18/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4442 - accuracy: 0.8311 - f1_m: 1.2453 - precision_m: 1.0967 - recall_m: 1.4463
Epoch 19/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4407 - accuracy: 0.8324 - f1_m: 1.2414 - precision_m: 1.0923 - recall_m: 1.4437
Epoch 20/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4391 - accuracy: 0.8332 - f1_m: 1.2420 - precision_m: 1.0933 - recall_m: 1.4442
Epoch 21/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4370 - accuracy: 0.8341 - f1_m: 1.2380 - precision_m: 1.0885 - recall_m: 1.4415
Epoch 22/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4338 - accuracy: 0.8348 - f1_m: 1.2392 - precision_m: 1.0893 - recall_m: 1.4429
Epoch 23/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4320 - accuracy: 0.8357 - f1_m: 1.2366 - precision_m: 1.0871 - recall_m: 1.4398
Epoch 24/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4310 - accuracy: 0.8355 - f1_m: 1.2334 - precision_m: 1.0832 - recall_m: 1.4380
Epoch 25/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4275 - accuracy: 0.8364 - f1_m: 1.2342 - precision_m: 1.0838 - recall_m: 1.4387
Epoch 26/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4263 - accuracy: 0.8371 - f1_m: 1.2341 - precision_m: 1.0839 - recall_m: 1.4385
Epoch 27/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4247 - accuracy: 0.8372 - f1_m: 1.2308 - precision_m: 1.0810 - recall_m: 1.4347
Epoch 28/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4227 - accuracy: 0.8383 - f1_m: 1.2309 - precision_m: 1.0806 - recall_m: 1.4355
Epoch 29/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4212 - accuracy: 0.8385 - f1_m: 1.2291 - precision_m: 1.0783 - recall_m: 1.4348
Epoch 30/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4207 - accuracy: 0.8387 - f1_m: 1.2243 - precision_m: 1.0733 - recall_m: 1.4306
Epoch 31/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4179 - accuracy: 0.8396 - f1_m: 1.2252 - precision_m: 1.0741 - recall_m: 1.4317
Epoch 32/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4179 - accuracy: 0.8395 - f1_m: 1.2269 - precision_m: 1.0761 - recall_m: 1.4328
Epoch 33/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4161 - accuracy: 0.8407 - f1_m: 1.2244 - precision_m: 1.0739 - recall_m: 1.4301
Epoch 34/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4147 - accuracy: 0.8404 - f1_m: 1.2227 - precision_m: 1.0715 - recall_m: 1.4293
Epoch 35/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4133 - accuracy: 0.8414 - f1_m: 1.2214 - precision_m: 1.0697 - recall_m: 1.4291
Epoch 36/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4125 - accuracy: 0.8424 - f1_m: 1.2213 - precision_m: 1.0698 - recall_m: 1.4287
Epoch 37/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4115 - accuracy: 0.8418 - f1_m: 1.2224 - precision_m: 1.0713 - recall_m: 1.4296
Epoch 38/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4094 - accuracy: 0.8425 - f1_m: 1.2206 - precision_m: 1.0686 - recall_m: 1.4289
Epoch 39/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4088 - accuracy: 0.8434 - f1_m: 1.2186 - precision_m: 1.0672 - recall_m: 1.4264
Epoch 40/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4075 - accuracy: 0.8435 - f1_m: 1.2177 - precision_m: 1.0660 - recall_m: 1.4259
Epoch 41/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4076 - accuracy: 0.8430 - f1_m: 1.2171 - precision_m: 1.0649 - recall_m: 1.4258
Epoch 42/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4057 - accuracy: 0.8437 - f1_m: 1.2151 - precision_m: 1.0632 - recall_m: 1.4232
Epoch 43/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4054 - accuracy: 0.8435 - f1_m: 1.2163 - precision_m: 1.0644 - recall_m: 1.4245
Epoch 44/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4043 - accuracy: 0.8439 - f1_m: 1.2150 - precision_m: 1.0629 - recall_m: 1.4236
Epoch 45/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4022 - accuracy: 0.8451 - f1_m: 1.2150 - precision_m: 1.0631 - recall_m: 1.4231
Epoch 46/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4026 - accuracy: 0.8448 - f1_m: 1.2153 - precision_m: 1.0639 - recall_m: 1.4235
Epoch 47/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4019 - accuracy: 0.8449 - f1_m: 1.2117 - precision_m: 1.0594 - recall_m: 1.4210
Epoch 48/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3998 - accuracy: 0.8461 - f1_m: 1.2125 - precision_m: 1.0603 - recall_m: 1.4216
Epoch 49/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.4000 - accuracy: 0.8455 - f1_m: 1.2132 - precision_m: 1.0604 - recall_m: 1.4233
Epoch 50/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3990 - accuracy: 0.8463 - f1_m: 1.2118 - precision_m: 1.0595 - recall_m: 1.4214
Epoch 51/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3977 - accuracy: 0.8467 - f1_m: 1.2122 - precision_m: 1.0599 - recall_m: 1.4213
Epoch 52/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3982 - accuracy: 0.8458 - f1_m: 1.2112 - precision_m: 1.0591 - recall_m: 1.4201
Epoch 53/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3975 - accuracy: 0.8459 - f1_m: 1.2093 - precision_m: 1.0569 - recall_m: 1.4191
Epoch 54/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3967 - accuracy: 0.8465 - f1_m: 1.2117 - precision_m: 1.0592 - recall_m: 1.4213
Epoch 55/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3952 - accuracy: 0.8476 - f1_m: 1.2091 - precision_m: 1.0563 - recall_m: 1.4193
Epoch 56/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3950 - accuracy: 0.8475 - f1_m: 1.2092 - precision_m: 1.0559 - recall_m: 1.4205
Epoch 57/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3936 - accuracy: 0.8475 - f1_m: 1.2116 - precision_m: 1.0598 - recall_m: 1.4201
Epoch 58/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3936 - accuracy: 0.8476 - f1_m: 1.2089 - precision_m: 1.0564 - recall_m: 1.4185
Epoch 59/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3922 - accuracy: 0.8481 - f1_m: 1.2089 - precision_m: 1.0558 - recall_m: 1.4195
Epoch 60/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3917 - accuracy: 0.8482 - f1_m: 1.2073 - precision_m: 1.0547 - recall_m: 1.4170
Epoch 61/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3917 - accuracy: 0.8477 - f1_m: 1.2073 - precision_m: 1.0546 - recall_m: 1.4176
Epoch 62/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3908 - accuracy: 0.8483 - f1_m: 1.2081 - precision_m: 1.0556 - recall_m: 1.4178
Epoch 63/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3901 - accuracy: 0.8487 - f1_m: 1.2079 - precision_m: 1.0552 - recall_m: 1.4180
Epoch 64/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3895 - accuracy: 0.8485 - f1_m: 1.2063 - precision_m: 1.0535 - recall_m: 1.4165
Epoch 65/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3892 - accuracy: 0.8483 - f1_m: 1.2071 - precision_m: 1.0547 - recall_m: 1.4169
Epoch 66/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3881 - accuracy: 0.8493 - f1_m: 1.2044 - precision_m: 1.0511 - recall_m: 1.4158
Epoch 67/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3873 - accuracy: 0.8496 - f1_m: 1.2047 - precision_m: 1.0521 - recall_m: 1.4150
Epoch 68/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3875 - accuracy: 0.8498 - f1_m: 1.2027 - precision_m: 1.0491 - recall_m: 1.4147
Epoch 69/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3870 - accuracy: 0.8498 - f1_m: 1.2067 - precision_m: 1.0541 - recall_m: 1.4167
Epoch 70/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3862 - accuracy: 0.8499 - f1_m: 1.2018 - precision_m: 1.0484 - recall_m: 1.4137
Epoch 71/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3861 - accuracy: 0.8497 - f1_m: 1.2026 - precision_m: 1.0497 - recall_m: 1.4135
Epoch 72/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3850 - accuracy: 0.8500 - f1_m: 1.2035 - precision_m: 1.0504 - recall_m: 1.4149
Epoch 73/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3856 - accuracy: 0.8505 - f1_m: 1.2025 - precision_m: 1.0492 - recall_m: 1.4143
Epoch 74/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3840 - accuracy: 0.8504 - f1_m: 1.2026 - precision_m: 1.0494 - recall_m: 1.4138
Epoch 75/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3839 - accuracy: 0.8506 - f1_m: 1.2032 - precision_m: 1.0503 - recall_m: 1.4137
Epoch 76/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3849 - accuracy: 0.8508 - f1_m: 1.2017 - precision_m: 1.0486 - recall_m: 1.4127
Epoch 77/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3825 - accuracy: 0.8516 - f1_m: 1.2021 - precision_m: 1.0483 - recall_m: 1.4146
Epoch 78/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3818 - accuracy: 0.8510 - f1_m: 1.2024 - precision_m: 1.0494 - recall_m: 1.4130
Epoch 79/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3819 - accuracy: 0.8516 - f1_m: 1.2018 - precision_m: 1.0483 - recall_m: 1.4136
Epoch 80/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3806 - accuracy: 0.8517 - f1_m: 1.2015 - precision_m: 1.0483 - recall_m: 1.4122
Epoch 81/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3810 - accuracy: 0.8521 - f1_m: 1.1995 - precision_m: 1.0461 - recall_m: 1.4112
Epoch 82/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3807 - accuracy: 0.8521 - f1_m: 1.1997 - precision_m: 1.0462 - recall_m: 1.4119
Epoch 83/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3793 - accuracy: 0.8524 - f1_m: 1.1978 - precision_m: 1.0436 - recall_m: 1.4106
Epoch 84/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3806 - accuracy: 0.8515 - f1_m: 1.1993 - precision_m: 1.0464 - recall_m: 1.4104
Epoch 85/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3787 - accuracy: 0.8523 - f1_m: 1.1989 - precision_m: 1.0451 - recall_m: 1.4117
Epoch 86/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3789 - accuracy: 0.8534 - f1_m: 1.1986 - precision_m: 1.0449 - recall_m: 1.4113
Epoch 87/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3790 - accuracy: 0.8522 - f1_m: 1.1970 - precision_m: 1.0434 - recall_m: 1.4093
Epoch 88/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3783 - accuracy: 0.8525 - f1_m: 1.1971 - precision_m: 1.0437 - recall_m: 1.4091
Epoch 89/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3782 - accuracy: 0.8528 - f1_m: 1.1974 - precision_m: 1.0436 - recall_m: 1.4097
Epoch 90/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3773 - accuracy: 0.8533 - f1_m: 1.1959 - precision_m: 1.0421 - recall_m: 1.4084
Epoch 91/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3757 - accuracy: 0.8532 - f1_m: 1.1967 - precision_m: 1.0433 - recall_m: 1.4087
Epoch 92/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3773 - accuracy: 0.8536 - f1_m: 1.1949 - precision_m: 1.0406 - recall_m: 1.4083
Epoch 93/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3753 - accuracy: 0.8535 - f1_m: 1.1955 - precision_m: 1.0415 - recall_m: 1.4081
Epoch 94/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3764 - accuracy: 0.8527 - f1_m: 1.1945 - precision_m: 1.0404 - recall_m: 1.4078
Epoch 95/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3754 - accuracy: 0.8536 - f1_m: 1.1960 - precision_m: 1.0426 - recall_m: 1.4083
Epoch 96/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3751 - accuracy: 0.8538 - f1_m: 1.1957 - precision_m: 1.0419 - recall_m: 1.4086
Epoch 97/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3743 - accuracy: 0.8539 - f1_m: 1.1955 - precision_m: 1.0416 - recall_m: 1.4084
Epoch 98/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3748 - accuracy: 0.8538 - f1_m: 1.1968 - precision_m: 1.0435 - recall_m: 1.4086
Epoch 99/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3735 - accuracy: 0.8544 - f1_m: 1.1962 - precision_m: 1.0426 - recall_m: 1.4088
Epoch 100/100
1349/1349 [==============================] - 2s 2ms/step - loss: 0.3739 - accuracy: 0.8547 - f1_m: 1.1936 - precision_m: 1.0393 - recall_m: 1.4075
2658/2658 [==============================] - 2s 873us/step
2658/2658 [==============================] - 3s 1ms/step - loss: 0.4152 - accuracy: 0.8446 - f1_m: 1.2172 - precision_m: 1.0753 - recall_m: 1.4248
score on test: 0.8446158766746521
5395/5395 [==============================] - 7s 1ms/step - loss: 0.3636 - accuracy: 0.8580 - f1_m: 1.2081 - precision_m: 1.0655 - recall_m: 1.4177
score on train: 0.8580316305160522

fig, c_ax = plt.subplots(1,1, figsize = (12, 8))
print('ROC AUC score:', multiclass_roc_auc_score(y_multi_test, y_pred))

c_ax.legend()
c_ax.set_xlabel('False Positive Rate')
c_ax.set_ylabel('True Positive Rate')
plt.show()

ROC AUC score: 0.7555054278721189

Accuracies plots

import plotly.express as px
levels = list(res_log.keys())
accuracies = list(res_log.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the logistic regression')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_rf_b.keys())
accuracies = list(res_rf_b.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the random forest for binary task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_xgb_b.keys())
accuracies = list(res_xgb_b.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the gradient boosting for binary task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_nn_b.keys())
accuracies = list(res_nn_b.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the neural network for binary task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_tree.keys())
accuracies = list(res_tree.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the decision tree')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_rf_m.keys())
accuracies = list(res_rf_m.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the random forest for multiclass task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_xgb_m.keys())
accuracies = list(res_xgb_m.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the gradient boosting for multiclass task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()

levels = list(res_nn_m.keys())
accuracies = list(res_nn_m.values())
finalplot=px.scatter(x=levels,y=accuracies,title='Accuracy levels in the neural network for multiclass task')
finalplot.update_layout(xaxis_tickangle=0, xaxis_title = 'Noise range',yaxis_title='Accuracy (%)')
finalplot.update_layout(uniformtext_minsize=0)
finalplot.show()